From 0e20c076b002ef87f3a7d12b2c791d55b6e4c05c Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 16 Jun 2017 14:20:36 -0700 Subject: [PATCH 01/84] First skeleton of Azure DNS Provider class --- octodns/provider/azuredns.py | 62 ++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 octodns/provider/azuredns.py diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py new file mode 100644 index 0000000..17e93b9 --- /dev/null +++ b/octodns/provider/azuredns.py @@ -0,0 +1,62 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from azure.common.credentials import ServicePrincipalCredentials +from azure.mgmt.dns import DnsManagementClient + + +from collections import defaultdict +# from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. +import logging +import re + +from ..record import Record, Update +from .base import BaseProvider + +class AzureProvider(BaseProvider): + ''' + Azure DNS Provider + + azure.py: + class: octodns.provider.azure.AzureProvider + # Current support of authentication of access to Azure services only + # includes using a Service Principal: + # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ + # resource-group-create-service-principal-portal + # The Azure Active Directory Application ID (referred to client ID) req: + client_id: + # Authentication Key Value req: + key: + # Directory ID (referred to tenant ID) req: + directory_id: + # Subscription ID req: + sub_id: + # Resource Group name req: + resource_group: + + testing: test authentication vars located in /home/t-hehwan/vars.txt + ''' + + # TODO. Will add support as project progresses. + SUPPORTS_GEO = False + + def __init__(self, id, client_id, key, directory_id, sub_id, \ + resource_group, *args, **kwargs): + self.log = logging.getLogger('AzureProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, client_id=%s, ' + 'key=***, directory_id:%s', id, client_id, directory_id) + super(AzureProvider, self).__init__(id, *args, **kwargs) + + credentials = ServicePrincipalCredentials( + client_id = client_id, secret = key, tenant = directory_id + ) + self._dns_client = DnsManagementClient(credentials, sub_id) + self._resource_group = resource_group + + + def _apply(self, plan): + From 386ada34f0750922a930fc0660f668d3552d6aae Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 19 Jun 2017 17:04:25 -0700 Subject: [PATCH 02/84] Added onto azuredns.py. Still completing code skeleton --- octodns/provider/azuredns.py | 116 +++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 13 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 17e93b9..8e1b992 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient - +from azure.mgmt.dns.models import * from collections import defaultdict # from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. @@ -17,6 +17,23 @@ import re from ..record import Record, Update from .base import BaseProvider +# Only made for A records. will have to adjust for more generic params types +class _AzureRecord(object): + def __init__(self, resource_group_name, record, values=None) + self.resource_group_name = resource_group_name + self.zone_name = record.zone.name + self.relative_record_set_name = record.name + self.record_type = record._type + + type_name = '{}records'.format(self.record_type) + class_name = '{}'.format(self.record_type).capitalize() + + 'Record'.format(self.record_type) + _values = [record._process_values] + self.params = {'ttl':record.ttl or 1800, \ + type_name:[eval(class_name)(value) for value in _values] or []} + + + class AzureProvider(BaseProvider): ''' Azure DNS Provider @@ -44,19 +61,92 @@ class AzureProvider(BaseProvider): # TODO. Will add support as project progresses. SUPPORTS_GEO = False - def __init__(self, id, client_id, key, directory_id, sub_id, \ - resource_group, *args, **kwargs): - self.log = logging.getLogger('AzureProvider[{}]'.format(id)) - self.log.debug('__init__: id=%s, client_id=%s, ' + def __init__(self, id, client_id, key, directory_id, sub_id, \ + resource_group, *args, **kwargs): + self.log = logging.getLogger('AzureProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, client_id=%s, ' 'key=***, directory_id:%s', id, client_id, directory_id) - super(AzureProvider, self).__init__(id, *args, **kwargs) + super(AzureProvider, self).__init__(id, *args, **kwargs) - credentials = ServicePrincipalCredentials( - client_id = client_id, secret = key, tenant = directory_id - ) - self._dns_client = DnsManagementClient(credentials, sub_id) - self._resource_group = resource_group + credentials = ServicePrincipalCredentials( + client_id = client_id, secret = key, tenant = directory_id + ) + self._dns_client = DnsManagementClient(credentials, sub_id) + self._resource_group = resource_group + + + self._azure_zones = None + self._azure_records = {} # this is populated through populate() + + # TODO: health checks a la route53. - def _apply(self, plan): - + + # TODO: add support for all types. First skeleton: add A. + def supports(self, record): + return record._type == 'A' + + @property + def azure_zones(self): + # TODO: return zones. will be created by populate() + + # Given a zone name, returns the zone id. If DNE, creates it. + def _get_zone_id(self, name): + + + def populate(self, zone, target): + self._azure_records = {} + + for record in zone.records: + + + def _apply_Create(self, change): + new = change.new + ar = self._get_azure_record(new) + + create = self._dns_client.record_sets.create_or_update + create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name \ + ar.record_type, ar.params) + + # type plan: Plan class from .base + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + + # validate that the zone exists. function creates zone if DNE. + self._get_zone_id(desired.name) + + + + # Some parsing bits to call _mod_Create or _mod_Delete. + # changes is a list of Delete and Create objects. + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + + + # ********** + # Figuring out what object plan is. + + # self._executor = ThreadPoolExecutor(max_workers) + # futures.append(self._executor.submit(self._populate_and_plan, + # zone_name, sources, targets)) + # plans = [p for f in futures for p in f.results()] + # type of plans[0] == type of one output of _populate_and_plan + + # for target, plan in plans: + # apply(plan) + + + # type(target) == BaseProvider + # type(plan) == Plan() + + # Plan(existing, desired, changes) + # existing.type == desired.type == Zone(desired.name, desired.sub_zones) + # Zone(name, sub_zones) (str and set of strs) + # changes.type = [Delete/Create] + \ No newline at end of file From ae9dd97f16e06152d27f28098b56a0f91eb476a5 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 20 Jun 2017 14:43:51 -0700 Subject: [PATCH 03/84] Filled out skeleton. Starting Testing --- .gitignore | 3 ++ MakeFile | 5 +++ octodns/provider/azuredns.py | 66 ++++++++++++++++++++++++++++++------ 3 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 MakeFile diff --git a/.gitignore b/.gitignore index 842a688..eca95c9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ nosetests.xml octodns.egg-info/ output/ tmp/ +Makefile +build/ +config/ \ No newline at end of file diff --git a/MakeFile b/MakeFile new file mode 100644 index 0000000..53fc947 --- /dev/null +++ b/MakeFile @@ -0,0 +1,5 @@ +local-rebuild: + sudo rm -r build + sudo rm -r octodns.egg-info/ + sudo python setup.py build -q + sudo python setup.py install -q \ No newline at end of file diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 8e1b992..2c55d7f 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -19,14 +19,14 @@ from .base import BaseProvider # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group_name, record, values=None) + def __init__(self, resource_group_name, record, values=None): self.resource_group_name = resource_group_name self.zone_name = record.zone.name self.relative_record_set_name = record.name self.record_type = record._type type_name = '{}records'.format(self.record_type) - class_name = '{}'.format(self.record_type).capitalize() + + class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) _values = [record._process_values] self.params = {'ttl':record.ttl or 1800, \ @@ -75,30 +75,76 @@ class AzureProvider(BaseProvider): self._resource_group = resource_group - self._azure_zones = None - self._azure_records = {} # this is populated through populate() + self._azure_zones = None # will be a dictionary. key: name. val: id. + self._azure_records = None # will be dict by octodns record, az record + self._supported_types = ['A'] + # TODO: health checks a la route53. # TODO: add support for all types. First skeleton: add A. def supports(self, record): - return record._type == 'A' + # TODO: possibly refactor + return record._type in self._supported_types @property def azure_zones(self): - # TODO: return zones. will be created by populate() + if self._azure_zones is None: + self.log.debug('azure_zones: loading') + zones = {} + for zone in self._dns_client.zones.list(): + zones[zone['name']] = zone['id'] + self._azure_zones = zones + return self._azure_zones # Given a zone name, returns the zone id. If DNE, creates it. - def _get_zone_id(self, name): - + def _get_zone_id(self, name, create=False): + self.log.debug('_get_zone_id: name=%s', name) + if name in self.azure_zones: + id = self.azure_zones[name] + self.log.debug('_get_zone_id: id=%s', id) + return id + if create: + #TODO + return None + # Create a dictionary of record objects by zone and octodns record names + # TODO: add geo parsing def populate(self, zone, target): - self._azure_records = {} + self.log.debug('populate: name=%s', zone.name) + before = len(zone.records) - for record in zone.records: + zone_id = self._get_zone_id(zone.name) + if zone_id: + records = defaultdict(list) + for type in self._supported_types: + for azrecord in self.dns_client.record_sets.list_by_type(self._resource_group, zone.name, type): + record_name = azrecord.name + data = getattr(self, '_data_for_{}'.format(type))(type, azrecord) + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) + self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) + + self.log.info('populate: found %s records', len(zone.records)-before) + + # might not need + def _get_type(azrecord): + azrecord['type'].split('/')[-1] + + def _data_for_A(self, type, azrecord): + return { + 'type': type + 'ttl': azrecord['ttl'], + 'values': [ar.ipv4_address for ar in azrecord.arecords] + } + def _get_azure_record(record): + try: + return self._azure_records[record] + except: + raise def _apply_Create(self, change): new = change.new From f48ef28688cc7be2211e42d4a39614fbb4cccb40 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 20 Jun 2017 14:55:59 -0700 Subject: [PATCH 04/84] Added shell script --- MakeFile | 5 ----- octodns/provider/azuredns.py | 3 +++ rb.txt | 7 +++++++ 3 files changed, 10 insertions(+), 5 deletions(-) delete mode 100644 MakeFile create mode 100755 rb.txt diff --git a/MakeFile b/MakeFile deleted file mode 100644 index 53fc947..0000000 --- a/MakeFile +++ /dev/null @@ -1,5 +0,0 @@ -local-rebuild: - sudo rm -r build - sudo rm -r octodns.egg-info/ - sudo python setup.py build -q - sudo python setup.py install -q \ No newline at end of file diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 2c55d7f..da348c1 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -17,6 +17,9 @@ import re from ..record import Record, Update from .base import BaseProvider + +#TODO: changes made to master include adding /build, Makefile to .gitignore and +# making Makefile. # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): def __init__(self, resource_group_name, record, values=None): diff --git a/rb.txt b/rb.txt new file mode 100755 index 0000000..30d33a1 --- /dev/null +++ b/rb.txt @@ -0,0 +1,7 @@ +#!/bin/bash +#script to rebuild octodns quickly + +sudo rm -r /home/t-hehwan/GitHub/octodns/build +sudo rm -r /home/t-hehwan/GitHub/octodns/octodns.egg-info +sudo python /home/t-hehwan/GitHub/octodns/setup.py build -q +sudo python /home/t-hehwan/GitHub/octodns/setup.py install -q From f5bce43e1054fe310dc77e379797149360a93326 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 20 Jun 2017 15:54:35 -0700 Subject: [PATCH 05/84] Testing AzureProvider. TODO: resolve 'Exception: Unknown provider class: octodns.provider.azure.AzureProvider' --- octodns/provider/azuredns.py | 85 ++++++++++++++++++------------------ rb.txt | 5 ++- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index da348c1..f1b342a 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -4,7 +4,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals - + from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient from azure.mgmt.dns.models import * @@ -13,10 +13,13 @@ from collections import defaultdict # from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. import logging import re +from ..record import Record, Up -from ..record import Record, Update from .base import BaseProvider +class A(BaseProvider): + def __init__(self): + pass #TODO: changes made to master include adding /build, Makefile to .gitignore and # making Makefile. @@ -38,60 +41,56 @@ class _AzureRecord(object): class AzureProvider(BaseProvider): - ''' - Azure DNS Provider - - azure.py: - class: octodns.provider.azure.AzureProvider - # Current support of authentication of access to Azure services only - # includes using a Service Principal: - # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ - # resource-group-create-service-principal-portal - # The Azure Active Directory Application ID (referred to client ID) req: - client_id: - # Authentication Key Value req: - key: - # Directory ID (referred to tenant ID) req: - directory_id: - # Subscription ID req: - sub_id: - # Resource Group name req: - resource_group: - - testing: test authentication vars located in /home/t-hehwan/vars.txt - ''' - - # TODO. Will add support as project progresses. - SUPPORTS_GEO = False - - def __init__(self, id, client_id, key, directory_id, sub_id, \ - resource_group, *args, **kwargs): + ''' + Azure DNS Provider + + azure.py: + class: octodns.provider.azure.AzureProvider + # Current support of authentication of access to Azure services only + # includes using a Service Principal: + # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ + # resource-group-create-service-principal-portal + # The Azure Active Directory Application ID (referred to client ID) req: + client_id: + # Authentication Key Value req: + key: + # Directory ID (referred to tenant ID) req: + directory_id: + # Subscription ID req: + sub_id: + # Resource Group name req: + resource_group: + + testing: test authentication vars located in /home/t-hehwan/vars.txt + ''' + SUPPORTS_GEO = False # TODO. Will add support as project progresses. + + def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, client_id=%s, ' - 'key=***, directory_id:%s', id, client_id, directory_id) + 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) - + credentials = ServicePrincipalCredentials( - client_id = client_id, secret = key, tenant = directory_id + client_id = client_id, secret = key, tenant = directory_id ) self._dns_client = DnsManagementClient(credentials, sub_id) self._resource_group = resource_group - self._azure_zones = None # will be a dictionary. key: name. val: id. self._azure_records = None # will be dict by octodns record, az record - + self._supported_types = ['A'] + + # TODO: health checks a la route53. + + - # TODO: health checks a la route53. - - - # TODO: add support for all types. First skeleton: add A. def supports(self, record): # TODO: possibly refactor return record._type in self._supported_types - + @property def azure_zones(self): if self._azure_zones is None: @@ -110,6 +109,7 @@ class AzureProvider(BaseProvider): self.log.debug('_get_zone_id: id=%s', id) return id if create: + raise Exception #TODO return None @@ -138,7 +138,7 @@ class AzureProvider(BaseProvider): def _data_for_A(self, type, azrecord): return { - 'type': type + 'type': type, 'ttl': azrecord['ttl'], 'values': [ar.ipv4_address for ar in azrecord.arecords] } @@ -154,8 +154,7 @@ class AzureProvider(BaseProvider): ar = self._get_azure_record(new) create = self._dns_client.record_sets.create_or_update - create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name \ - ar.record_type, ar.params) + create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name, ar.record_type, ar.params) # type plan: Plan class from .base def _apply(self, plan): diff --git a/rb.txt b/rb.txt index 30d33a1..d6a7949 100755 --- a/rb.txt +++ b/rb.txt @@ -3,5 +3,6 @@ sudo rm -r /home/t-hehwan/GitHub/octodns/build sudo rm -r /home/t-hehwan/GitHub/octodns/octodns.egg-info -sudo python /home/t-hehwan/GitHub/octodns/setup.py build -q -sudo python /home/t-hehwan/GitHub/octodns/setup.py install -q +sudo python /home/t-hehwan/GitHub/octodns/setup.py -q build +sudo python /home/t-hehwan/GitHub/octodns/setup.py -q install +octodns-sync --config-file=./config/production.yaml \ No newline at end of file From 92828ce1c6a99c6d29fe2b223cc9e938a404fae1 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Wed, 21 Jun 2017 14:21:14 -0700 Subject: [PATCH 06/84] Successfully able to add A records. TODO: check against live server to remove records not listed in config --- octodns/provider/azuredns.py | 85 +++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index f1b342a..3a442e8 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +import sys from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient @@ -13,32 +14,41 @@ from collections import defaultdict # from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. import logging import re -from ..record import Record, Up - +from ..record import Record, Update from .base import BaseProvider -class A(BaseProvider): - def __init__(self): - pass #TODO: changes made to master include adding /build, Makefile to .gitignore and # making Makefile. # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group_name, record, values=None): - self.resource_group_name = resource_group_name - self.zone_name = record.zone.name - self.relative_record_set_name = record.name + def __init__(self, resource_group, record, values=None): + self.resource_group = resource_group + self.zone_name = record.zone.name[0:len(record.zone.name)-1] # strips last period + self.relative_record_set_name = record.name or '@' self.record_type = record._type - type_name = '{}records'.format(self.record_type) + type_name = '{}records'.format(self.record_type).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) - _values = [record._process_values] - self.params = {'ttl':record.ttl or 1800, \ - type_name:[eval(class_name)(value) for value in _values] or []} + data = values or record.data #This should fail if it gets to record.data? It only returns ttl. TODO + print('findme: ',file=sys.stderr) + print(data, file=sys.stderr) + print('\n',file=sys.stderr) + #depending on mult values or not + self.params = {} + try: + self.params = {'ttl':record.ttl or 1800, \ + type_name:[eval(class_name)(ip) for ip in data['values']] or []} + except KeyError: # means that doesn't have multiple values but single value + self.params = {'ttl':record.ttl or 1800, \ + type_name:[eval(class_name)(data['value'])] or []} + + # ar = _AzureRecord(self._resource_group, new, new.data) + + class AzureProvider(BaseProvider): ''' @@ -96,8 +106,8 @@ class AzureProvider(BaseProvider): if self._azure_zones is None: self.log.debug('azure_zones: loading') zones = {} - for zone in self._dns_client.zones.list(): - zones[zone['name']] = zone['id'] + for zone in self._dns_client.zones.list_by_resource_group(self._resource_group): + zones[zone.name] = zone.id self._azure_zones = zones return self._azure_zones @@ -109,8 +119,9 @@ class AzureProvider(BaseProvider): self.log.debug('_get_zone_id: id=%s', id) return id if create: + self.log.debug('_get_zone_id: no matching zone; creating %s', name) raise Exception - #TODO + #TODO, write code return None # Create a dictionary of record objects by zone and octodns record names @@ -128,7 +139,7 @@ class AzureProvider(BaseProvider): data = getattr(self, '_data_for_{}'.format(type))(type, azrecord) record = Record.new(zone, record_name, data, source=self) zone.add_record(record) - self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) + self._azure_records[record] = _AzureRecord(self._resource_group, record, data) self.log.info('populate: found %s records', len(zone.records)-before) @@ -137,24 +148,48 @@ class AzureProvider(BaseProvider): azrecord['type'].split('/')[-1] def _data_for_A(self, type, azrecord): + print('xxxxx\n',file=sys.stderr) + for ar in azrecords.arecords: + print(ar,file=sys.stderr) + + + v = [ARecord(ar.ipv4_address) for ar in azrecord.arecords] + # print(v, file=sys.stderr) + # print('99999999999999999999999999999999999999999999999999\n',file=sys.stderr) return { 'type': type, 'ttl': azrecord['ttl'], - 'values': [ar.ipv4_address for ar in azrecord.arecords] + 'value': v } - def _get_azure_record(record): - try: - return self._azure_records[record] - except: - raise + # def _get_azure_record(self, record): + # try: + # return self._azure_records[record] + # except: + # self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) + # return self._azure_records[record] + # except: + # raise def _apply_Create(self, change): new = change.new - ar = self._get_azure_record(new) + + #validate that the zone exists. + #self._get_zone_id(new.name, create=True) + + #ar = self._get_azure_record(new) + ar = _AzureRecord(self._resource_group, new, new.data) create = self._dns_client.record_sets.create_or_update - create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name, ar.record_type, ar.params) + print('find:{} {} {} {} {}\n'.format(ar.resource_group,ar.zone_name,ar.relative_record_set_name,ar.record_type,ar.params), file=sys.stderr) + for arec in ar.params['arecords']: + print(str(arec.ipv4_address) + ', ', file=sys.stderr) + print('\n',file=sys.stderr) + create(resource_group_name=ar.resource_group, + zone_name=ar.zone_name, + relative_record_set_name=ar.relative_record_set_name, + record_type=ar.record_type, + parameters=ar.params) # type plan: Plan class from .base def _apply(self, plan): From 852c1013889e6c6c4504692e7e91948b364c22a7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 21 Jun 2017 17:08:16 -0700 Subject: [PATCH 07/84] Switch to an explicit SUPPORTS setup --- octodns/provider/cloudflare.py | 7 ++----- octodns/provider/dnsimple.py | 2 ++ octodns/provider/dyn.py | 2 ++ octodns/provider/ns1.py | 6 +++--- octodns/provider/powerdns.py | 2 ++ octodns/provider/route53.py | 5 ++--- octodns/provider/yaml.py | 2 ++ octodns/source/base.py | 7 ++++--- octodns/source/tinydns.py | 1 + tests/helpers.py | 1 + tests/test_octodns_provider_base.py | 13 +++++++++++-- 11 files changed, 32 insertions(+), 16 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index eb44d30..a45bd44 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -36,7 +36,7 @@ class CloudflareProvider(BaseProvider): ''' SUPPORTS_GEO = False # TODO: support SRV - UNSUPPORTED_TYPES = ('ALIAS', 'NAPTR', 'PTR', 'SOA', 'SRV', 'SSHFP') + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'SPF', 'TXT')) MIN_TTL = 120 TIMEOUT = 15 @@ -56,9 +56,6 @@ class CloudflareProvider(BaseProvider): self._zones = None self._zone_records = {} - def supports(self, record): - return record._type not in self.UNSUPPORTED_TYPES - def _request(self, method, path, params=None, data=None): self.log.debug('_request: method=%s, path=%s', method, path) @@ -167,7 +164,7 @@ class CloudflareProvider(BaseProvider): for record in records: name = zone.hostname_from_fqdn(record['name']) _type = record['type'] - if _type not in self.UNSUPPORTED_TYPES: + if _type in self.SUPPORTS: values[name][record['type']].append(record) for name, types in values.items(): diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 763e446..cb0f2d7 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -91,6 +91,8 @@ class DnsimpleProvider(BaseProvider): account: 42 ''' SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SPF', 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, token, account, *args, **kwargs): self.log = logging.getLogger('DnsimpleProvider[{}]'.format(id)) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index ac0e21b..673e8d0 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -106,6 +106,7 @@ class DynProvider(BaseProvider): than one account active at a time. See DynProvider._check_dyn_sess for some related bits. ''' + RECORDS_TO_TYPE = { 'a_records': 'A', 'aaaa_records': 'AAAA', @@ -121,6 +122,7 @@ class DynProvider(BaseProvider): 'txt_records': 'TXT', } TYPE_TO_RECORDS = {v: k for k, v in RECORDS_TO_TYPE.items()} + SUPPORTS = set(TYPE_TO_RECORDS.keys()) # https://help.dyn.com/predefined-geotm-regions-groups/ REGION_CODES = { diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 5a51780..c50341d 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -22,6 +22,9 @@ class Ns1Provider(BaseProvider): api_key: env/NS1_API_KEY ''' SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SPF', 'SRV', 'TXT')) + ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' def __init__(self, id, api_key, *args, **kwargs): @@ -30,9 +33,6 @@ class Ns1Provider(BaseProvider): super(Ns1Provider, self).__init__(id, *args, **kwargs) self._client = NSONE(apiKey=api_key) - def supports(self, record): - return record._type != 'SSHFP' - def _data_for_A(self, _type, record): return { 'ttl': record['ttl'], diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 4ff2568..21e4d44 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -14,6 +14,8 @@ from .base import BaseProvider class PowerDnsBaseProvider(BaseProvider): SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 def __init__(self, id, host, api_key, port=8081, *args, **kwargs): diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index bc8bc34..12c1aff 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -220,6 +220,8 @@ class Route53Provider(BaseProvider): In general the account used will need full permissions on Route53. ''' SUPPORTS_GEO = True + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', + 'SRV', 'TXT')) # This should be bumped when there are underlying changes made to the # health check config. @@ -239,9 +241,6 @@ class Route53Provider(BaseProvider): self._r53_rrsets = {} self._health_checks = None - def supports(self, record): - return record._type not in ('ALIAS', 'SSHFP') - @property def r53_zones(self): if self._r53_zones is None: diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 7b2d209..c728caf 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -31,6 +31,8 @@ class YamlProvider(BaseProvider): enforce_order: True ''' SUPPORTS_GEO = True + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, *args, **kwargs): diff --git a/octodns/source/base.py b/octodns/source/base.py index 72ebaab..42d214b 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -16,6 +16,9 @@ class BaseSource(object): if not hasattr(self, 'SUPPORTS_GEO'): raise NotImplementedError('Abstract base class, SUPPORTS_GEO ' 'property missing') + if not hasattr(self, 'SUPPORTS'): + raise NotImplementedError('Abstract base class, SUPPORTS ' + 'property missing') def populate(self, zone, target=False): ''' @@ -25,9 +28,7 @@ class BaseSource(object): 'missing') def supports(self, record): - # Unless overriden and handled appropriaitely we'll assume that all - # record types are supported - return True + return record._type in self.SUPPORTS def __repr__(self): return self.__class__.__name__ diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 70f0145..6805378 100644 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -19,6 +19,7 @@ from .base import BaseSource class TinyDnsBaseSource(BaseSource): SUPPORTS_GEO = False + SUPPORTS = set(('A', 'CNAME', 'MX', 'NS')) split_re = re.compile(r':+') diff --git a/tests/helpers.py b/tests/helpers.py index df74e84..a8aafa3 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -17,6 +17,7 @@ class SimpleSource(object): class SimpleProvider(object): SUPPORTS_GEO = False + SUPPORTS = set(('A',)) def __init__(self, id='test'): pass diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index c7836c8..766bf65 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -16,6 +16,8 @@ from octodns.zone import Zone class HelperProvider(BaseProvider): log = getLogger('HelperProvider') + SUPPORTS = set(('A',)) + def __init__(self, extra_changes, apply_disabled=False, include_change_callback=None): self.__extra_changes = extra_changes @@ -58,10 +60,17 @@ class TestBaseProvider(TestCase): zone = Zone('unit.tests.', []) with self.assertRaises(NotImplementedError) as ctx: HasSupportsGeo('hassupportesgeo').populate(zone) + self.assertEquals('Abstract base class, SUPPORTS property missing', + ctx.exception.message) + + class HasSupports(HasSupportsGeo): + SUPPORTS = set(('A',)) + with self.assertRaises(NotImplementedError) as ctx: + HasSupports('hassupportes').populate(zone) self.assertEquals('Abstract base class, populate method missing', ctx.exception.message) - class HasPopulate(HasSupportsGeo): + class HasPopulate(HasSupports): def populate(self, zone, target=False): zone.add_record(Record.new(zone, '', { @@ -81,7 +90,7 @@ class TestBaseProvider(TestCase): 'value': '1.2.3.4' })) - self.assertTrue(HasSupportsGeo('hassupportesgeo') + self.assertTrue(HasSupports('hassupportesgeo') .supports(list(zone.records)[0])) plan = HasPopulate('haspopulate').plan(zone) From 3c1e409e6fea2347c4cd5d18c6fccc84387ae797 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Thu, 22 Jun 2017 14:22:40 -0700 Subject: [PATCH 08/84] Added support for CNAME, AAAA, MX, SRV, NS, PTR. TODO: add TXT. add zone creation. create tests --- octodns/provider/azuredns.py | 162 ++++++++++++++++++++++------------- 1 file changed, 101 insertions(+), 61 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3a442e8..66ca8e3 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -22,7 +22,8 @@ from .base import BaseProvider # making Makefile. # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group, record, values=None): + def __init__(self, resource_group, record, values=None, ttl=1800): + # print('Here4',file=sys.stderr) self.resource_group = resource_group self.zone_name = record.zone.name[0:len(record.zone.name)-1] # strips last period self.relative_record_set_name = record.name or '@' @@ -31,22 +32,21 @@ class _AzureRecord(object): type_name = '{}records'.format(self.record_type).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) + if values == None: + return + # TODO: clean up this bit. data = values or record.data #This should fail if it gets to record.data? It only returns ttl. TODO - print('findme: ',file=sys.stderr) - print(data, file=sys.stderr) - print('\n',file=sys.stderr) #depending on mult values or not + #TODO: import explicitly. eval() uses for example ARecord from azure.mgmt.dns.models self.params = {} try: - self.params = {'ttl':record.ttl or 1800, \ + self.params = {'ttl':record.ttl or ttl, \ type_name:[eval(class_name)(ip) for ip in data['values']] or []} except KeyError: # means that doesn't have multiple values but single value - self.params = {'ttl':record.ttl or 1800, \ + self.params = {'ttl':record.ttl or ttl, \ type_name:[eval(class_name)(data['value'])] or []} - - # ar = _AzureRecord(self._resource_group, new, new.data) @@ -88,17 +88,17 @@ class AzureProvider(BaseProvider): self._resource_group = resource_group self._azure_zones = None # will be a dictionary. key: name. val: id. - self._azure_records = None # will be dict by octodns record, az record + self._azure_records = {} # will be dict by octodns record, az record - self._supported_types = ['A'] + self._supported_types = ['CNAME', 'A', 'AAAA', 'MX', 'SRV', 'NS', 'PTR'] + # TODO: add TXT - # TODO: health checks a la route53. + # TODO: health checks a la route53. # TODO: add support for all types. First skeleton: add A. def supports(self, record): - # TODO: possibly refactor return record._type in self._supported_types @property @@ -114,82 +114,121 @@ class AzureProvider(BaseProvider): # Given a zone name, returns the zone id. If DNE, creates it. def _get_zone_id(self, name, create=False): self.log.debug('_get_zone_id: name=%s', name) - if name in self.azure_zones: - id = self.azure_zones[name] + try: + id = self._dns_client.zones.get(self._resource_group, name) self.log.debug('_get_zone_id: id=%s', id) return id - if create: - self.log.debug('_get_zone_id: no matching zone; creating %s', name) - raise Exception - #TODO, write code - return None - + except: + if create: + self.log.debug('_get_zone_id: no matching zone; creating %s', name) + #TODO: write + return None #placeholder + return None + # Create a dictionary of record objects by zone and octodns record names # TODO: add geo parsing def populate(self, zone, target): - self.log.debug('populate: name=%s', zone.name) + zone_name = zone.name[0:len(zone.name)-1]#Azure zone names do not include suffix . + self.log.debug('populate: name=%s', zone_name) before = len(zone.records) - - zone_id = self._get_zone_id(zone.name) + zone_id = self._get_zone_id(zone_name) if zone_id: - records = defaultdict(list) + #records = defaultdict(list) for type in self._supported_types: - for azrecord in self.dns_client.record_sets.list_by_type(self._resource_group, zone.name, type): - record_name = azrecord.name - data = getattr(self, '_data_for_{}'.format(type))(type, azrecord) + # print('populate. type: {}'.format(type),file=sys.stderr) + for azrecord in self._dns_client.record_sets.list_by_type(self._resource_group, zone_name, type): + # print(azrecord, file=sys.stderr) + record_name = azrecord.name if azrecord.name != '@' else '' + data = self._type_and_ttl(type, azrecord, + getattr(self, '_data_for_{}'.format(type))(azrecord)) # TODO: azure online interface allows None values. must validate. record = Record.new(zone, record_name, data, source=self) + # print('HERE0',file=sys.stderr) zone.add_record(record) self._azure_records[record] = _AzureRecord(self._resource_group, record, data) - + # print('HERE1',file=sys.stderr) self.log.info('populate: found %s records', len(zone.records)-before) - # might not need + # might not need def _get_type(azrecord): azrecord['type'].split('/')[-1] - def _data_for_A(self, type, azrecord): - print('xxxxx\n',file=sys.stderr) - for ar in azrecords.arecords: - print(ar,file=sys.stderr) - + def _type_and_ttl(self, type, azrecord, data): + data['type'] = type + data['ttl'] = azrecord.ttl + return data - v = [ARecord(ar.ipv4_address) for ar in azrecord.arecords] - # print(v, file=sys.stderr) - # print('99999999999999999999999999999999999999999999999999\n',file=sys.stderr) - return { - 'type': type, - 'ttl': azrecord['ttl'], - 'value': v + def _data_for_A(self, azrecord): + return {'values': [ar.ipv4_address for ar in azrecord.arecords]} + + def _data_for_AAAA(self, azrecord): + return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} + + def _data_for_TXT(self, azrecord): + print('azure',file=sys.stderr) + print([ar.value for ar in azrecord.txt_records], file=sys.stderr) + print('',file=sys.stderr) + return {'values': [ar.value for ar in azrecord.txt_records]} + + def _data_for_CNAME(self, azrecord): + try: + val = azrecord.cname_record.cname + if not val.endswith('.'): + val += '.' + return {'value': val} + except: + return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. + + def _data_for_PTR(self, azrecord): + try: + val = azrecord.ptr_records[0].ptdrname + if not val.endswith('.'): + val += '.' + return {'value': val} + except: + return {'value': '.' } #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. + + def _data_for_MX(self, azrecord): + return {'values': [{'priority':ar.preference, + 'value':ar.exchange} for ar in azrecord.mx_records]} + + def _data_for_SRV(self, azrecord): + return {'values': [{'priority': ar.priority, + 'weight': ar.weight, + 'port': ar.port, + 'target': ar.target} for ar in azrecord.srv_records] } - - # def _get_azure_record(self, record): - # try: - # return self._azure_records[record] - # except: - # self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) - # return self._azure_records[record] - # except: - # raise - + + def _data_for_NS(self, azrecord): + def period_validate(string): + return string if string.endswith('.') else string + '.' + vals = [ar.nsdname for ar in azrecord.ns_records] + return {'values': [period_validate(val) for val in vals]} + def _apply_Create(self, change): new = change.new - + #validate that the zone exists. #self._get_zone_id(new.name, create=True) - #ar = self._get_azure_record(new) ar = _AzureRecord(self._resource_group, new, new.data) create = self._dns_client.record_sets.create_or_update - print('find:{} {} {} {} {}\n'.format(ar.resource_group,ar.zone_name,ar.relative_record_set_name,ar.record_type,ar.params), file=sys.stderr) - for arec in ar.params['arecords']: - print(str(arec.ipv4_address) + ', ', file=sys.stderr) - print('\n',file=sys.stderr) create(resource_group_name=ar.resource_group, zone_name=ar.zone_name, relative_record_set_name=ar.relative_record_set_name, record_type=ar.record_type, parameters=ar.params) + + def _apply_Delete(self, change): + existing = change.existing + ar = _AzureRecord(self._resource_group, existing) + delete = self._dns_client.record_sets.delete + delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, + ar.record_type) + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) # type plan: Plan class from .base def _apply(self, plan): @@ -198,12 +237,9 @@ class AzureProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - # validate that the zone exists. function creates zone if DNE. self._get_zone_id(desired.name) - - # Some parsing bits to call _mod_Create or _mod_Delete. # changes is a list of Delete and Create objects. for change in changes: @@ -232,4 +268,8 @@ class AzureProvider(BaseProvider): # existing.type == desired.type == Zone(desired.name, desired.sub_zones) # Zone(name, sub_zones) (str and set of strs) # changes.type = [Delete/Create] - \ No newline at end of file + + + # Starts with sync in main() of sync. + # {u'values': ['3.3.3.3', '4.4.4.4'], u'type': 'A', u'ttl': 3600} + # {u'type': u'A', u'value': [u'3.3.3.3', u'4.4.4.4'], u'ttl': 3600L} \ No newline at end of file From 8323b4c0ea134c45cddcf00e1667125524a7dbe6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 07:14:01 -0700 Subject: [PATCH 09/84] Complete refactor & rework of how validation is set up This is with an eye toward expanding it in the future both in terms of what it checks and to add the ability to ignore things. This commit does not intend to change any validation. It only reworks the flow and improves the error messaging. --- octodns/record.py | 350 ++++++++++----- tests/test_octodns_record.py | 807 ++++++++++++++++++++++++++++------- 2 files changed, 895 insertions(+), 262 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index cacb147..827ad0a 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -54,7 +54,14 @@ class Delete(Change): return 'Delete {}'.format(self.existing) -_unescaped_semicolon_re = re.compile(r'\w;') +class ValidationError(Exception): + + def __init__(self, fqdn, reasons): + message = 'Invalid record {}\n - {}' \ + .format(fqdn, '\n - '.join(reasons)) + super(Exception, self).__init__(message) + self.fqdn = fqdn + self.reasons = reasons class Record(object): @@ -62,13 +69,13 @@ class Record(object): @classmethod def new(cls, zone, name, data, source=None): + fqdn = '{}.{}'.format(name, zone.name) if name else zone.name try: _type = data['type'] except KeyError: - fqdn = '{}.{}'.format(name, zone.name) if name else zone.name raise Exception('Invalid record {}, missing type'.format(fqdn)) try: - _type = { + _class = { 'A': ARecord, 'AAAA': AaaaRecord, 'ALIAS': AliasRecord, @@ -98,7 +105,21 @@ class Record(object): }[_type] except KeyError: raise Exception('Unknown record type: "{}"'.format(_type)) - return _type(zone, name, data, source=source) + reasons = _class.validate(name, data) + if reasons: + raise ValidationError(fqdn, reasons) + return _class(zone, name, data, source=source) + + @classmethod + def validate(cls, name, data): + reasons = [] + try: + ttl = int(data['ttl']) + if ttl < 0: + reasons.append('invalid ttl') + except KeyError: + reasons.append('missing ttl') + return reasons def __init__(self, zone, name, data, source=None): self.log.debug('__init__: zone.name=%s, type=%11s, name=%s', zone.name, @@ -106,11 +127,8 @@ class Record(object): self.zone = zone # force everything lower-case just to be safe self.name = str(name).lower() if name else name - try: - self.ttl = int(data['ttl']) - except KeyError: - raise Exception('Invalid record {}, missing ttl'.format(self.fqdn)) self.source = source + self.ttl = int(data['ttl']) octodns = data.get('octodns', {}) self.ignored = octodns.get('ignored', False) @@ -154,11 +172,17 @@ class GeoValue(object): geo_re = re.compile(r'^(?P\w\w)(-(?P\w\w)' r'(-(?P\w\w))?)?$') - def __init__(self, geo, values): - match = self.geo_re.match(geo) + @classmethod + def _validate_geo(cls, code): + reasons = [] + match = cls.geo_re.match(code) if not match: - raise Exception('Invalid geo "{}"'.format(geo)) + reasons.append('invalid geo "{}"'.format(code)) + return reasons + + def __init__(self, geo, values): self.code = geo + match = self.geo_re.match(geo) self.continent_code = match.group('continent_code') self.country_code = match.group('country_code') self.subdivision_code = match.group('subdivision_code') @@ -185,16 +209,29 @@ class GeoValue(object): class _ValuesMixin(object): - def __init__(self, zone, name, data, source=None): - super(_ValuesMixin, self).__init__(zone, name, data, source=source) + @classmethod + def validate(cls, name, data): + reasons = super(_ValuesMixin, cls).validate(name, data) + values = [] try: values = data['values'] except KeyError: try: values = [data['value']] except KeyError: - raise Exception('Invalid record {}, missing value(s)' - .format(self.fqdn)) + reasons.append('missing value(s)') + + for value in values: + reasons.extend(cls._validate_value(value)) + + return reasons + + def __init__(self, zone, name, data, source=None): + super(_ValuesMixin, self).__init__(zone, name, data, source=source) + try: + values = data['values'] + except KeyError: + values = [data['value']] self.values = sorted(self._process_values(values)) def changes(self, other, target): @@ -224,6 +261,21 @@ class _GeoMixin(_ValuesMixin): Must be included before `Record`. ''' + @classmethod + def validate(cls, name, data): + reasons = super(_GeoMixin, cls).validate(name, data) + try: + geo = dict(data['geo']) + # TODO: validate legal codes + for code, values in geo.items(): + reasons.extend(GeoValue._validate_geo(code)) + for value in values: + reasons.extend(cls._validate_value(value)) + except KeyError: + pass + return reasons + + # TODO: support 'value' as well # TODO: move away from "data" hash to strict params, it's kind of leaking # the yaml implementation into here and then forcing it back out into # non-yaml providers during input @@ -233,9 +285,8 @@ class _GeoMixin(_ValuesMixin): self.geo = dict(data['geo']) except KeyError: self.geo = {} - for k, vs in self.geo.items(): - vs = sorted(self._process_values(vs)) - self.geo[k] = GeoValue(k, vs) + for code, values in self.geo.items(): + self.geo[code] = GeoValue(code, values) def _data(self): ret = super(_GeoMixin, self)._data() @@ -264,41 +315,52 @@ class _GeoMixin(_ValuesMixin): class ARecord(_GeoMixin, Record): _type = 'A' + @classmethod + def _validate_value(self, value): + reasons = [] + try: + IPv4Address(unicode(value)) + except Exception: + reasons.append('invalid ip address "{}"'.format(value)) + return reasons + def _process_values(self, values): - for ip in values: - try: - IPv4Address(unicode(ip)) - except Exception: - raise Exception('Invalid record {}, value {} not a valid ip' - .format(self.fqdn, ip)) return values class AaaaRecord(_GeoMixin, Record): _type = 'AAAA' + @classmethod + def _validate_value(self, value): + reasons = [] + try: + IPv6Address(unicode(value)) + except Exception: + reasons.append('invalid ip address "{}"'.format(value)) + return reasons + def _process_values(self, values): - ret = [] - for ip in values: - try: - IPv6Address(unicode(ip)) - ret.append(ip.lower()) - except Exception: - raise Exception('Invalid record {}, value {} not a valid ip' - .format(self.fqdn, ip)) - return ret + return values class _ValueMixin(object): - def __init__(self, zone, name, data, source=None): - super(_ValueMixin, self).__init__(zone, name, data, source=source) + @classmethod + def validate(cls, name, data): + reasons = super(_ValueMixin, cls).validate(name, data) + value = None try: value = data['value'] except KeyError: - raise Exception('Invalid record {}, missing value' - .format(self.fqdn)) - self.value = self._process_value(value) + reasons.append('missing value') + if value: + reasons.extend(cls._validate_value(value)) + return reasons + + def __init__(self, zone, name, data, source=None): + super(_ValueMixin, self).__init__(zone, name, data, source=source) + self.value = self._process_value(data['value']) def changes(self, other, target): if self.value != other.value: @@ -319,25 +381,42 @@ class _ValueMixin(object): class AliasRecord(_ValueMixin, Record): _type = 'ALIAS' - def _process_value(self, value): + @classmethod + def _validate_value(self, value): + reasons = [] if not value.endswith('.'): - raise Exception('Invalid record {}, value ({}) missing trailing .' - .format(self.fqdn, value)) + reasons.append('missing trailing .') + return reasons + + def _process_value(self, value): return value class CnameRecord(_ValueMixin, Record): _type = 'CNAME' - def _process_value(self, value): + @classmethod + def _validate_value(cls, value): + reasons = [] if not value.endswith('.'): - raise Exception('Invalid record {}, value ({}) missing trailing .' - .format(self.fqdn, value)) - return value.lower() + reasons.append('missing trailing .') + return reasons + + def _process_value(self, value): + return value class MxValue(object): + @classmethod + def _validate_value(cls, value): + reasons = [] + if 'priority' not in value: + reasons.append('missing priority') + if 'value' not in value: + reasons.append('missing value') + return reasons + def __init__(self, value): # TODO: rename preference self.priority = int(value['priority']) @@ -363,19 +442,38 @@ class MxValue(object): class MxRecord(_ValuesMixin, Record): _type = 'MX' + @classmethod + def _validate_value(cls, value): + return MxValue._validate_value(value) + def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(MxValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [MxValue(v) for v in values] class NaptrValue(object): + @classmethod + def _validate_value(cls, data): + reasons = [] + try: + int(data['order']) + except KeyError: + reasons.append('missing order') + except ValueError: + reasons.append('invalid order "{}"'.format(data['order'])) + try: + int(data['preference']) + except KeyError: + reasons.append('missing preference') + except ValueError: + reasons.append('invalid preference "{}"' + .format(data['preference'])) + # TODO: validate field data + for k in ('flags', 'service', 'regexp', 'replacement'): + if k not in data: + reasons.append('missing {}'.format(k)) + return reasons + def __init__(self, value): self.order = int(value['order']) self.preference = int(value['preference']) @@ -420,42 +518,65 @@ class NaptrValue(object): class NaptrRecord(_ValuesMixin, Record): _type = 'NAPTR' + @classmethod + def _validate_value(cls, value): + return NaptrValue._validate_value(value) + def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(NaptrValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [NaptrValue(v) for v in values] class NsRecord(_ValuesMixin, Record): _type = 'NS' + @classmethod + def _validate_value(cls, value): + reasons = [] + if not value.endswith('.'): + reasons.append('missing trailing .') + return reasons + def _process_values(self, values): - ret = [] - for ns in values: - if not ns.endswith('.'): - raise Exception('Invalid record {}, value {} missing ' - 'trailing .'.format(self.fqdn, ns)) - ret.append(ns.lower()) - return ret + return values class PtrRecord(_ValueMixin, Record): _type = 'PTR' - def _process_value(self, value): + @classmethod + def _validate_value(cls, value): + reasons = [] if not value.endswith('.'): - raise Exception('Invalid record {}, value ({}) missing trailing .' - .format(self.fqdn, value)) - return value.lower() + reasons.append('missing trailing .') + return reasons + + def _process_value(self, value): + return value class SshfpValue(object): + @classmethod + def _validate_value(cls, value): + reasons = [] + # TODO: validate algorithm and fingerprint_type values + try: + int(value['algorithm']) + except KeyError: + reasons.append('missing algorithm') + except ValueError: + reasons.append('invalid algorithm "{}"'.format(value['algorithm'])) + try: + int(value['fingerprint_type']) + except KeyError: + reasons.append('missing fingerprint_type') + except ValueError: + reasons.append('invalid fingerprint_type "{}"' + .format(value['fingerprint_type'])) + if 'fingerprint' not in value: + reasons.append('missing fingerprint') + return reasons + def __init__(self, value): self.algorithm = int(value['algorithm']) self.fingerprint_type = int(value['fingerprint_type']) @@ -484,26 +605,61 @@ class SshfpValue(object): class SshfpRecord(_ValuesMixin, Record): _type = 'SSHFP' + @classmethod + def _validate_value(cls, value): + return SshfpValue._validate_value(value) + def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(SshfpValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [SshfpValue(v) for v in values] + + +_unescaped_semicolon_re = re.compile(r'\w;') class SpfRecord(_ValuesMixin, Record): _type = 'SPF' + @classmethod + def _validate_value(cls, value): + if _unescaped_semicolon_re.search(value): + return ['unescaped ;'] + return [] + def _process_values(self, values): return values class SrvValue(object): + @classmethod + def _validate_value(self, value): + reasons = [] + # TODO: validate algorithm and fingerprint_type values + try: + int(value['priority']) + except KeyError: + reasons.append('missing priority') + except ValueError: + reasons.append('invalid priority "{}"'.format(value['priority'])) + try: + int(value['weight']) + except KeyError: + reasons.append('missing weight') + except ValueError: + reasons.append('invalid weight "{}"'.format(value['weight'])) + try: + int(value['port']) + except KeyError: + reasons.append('missing port') + except ValueError: + reasons.append('invalid port "{}"'.format(value['port'])) + try: + if not value['target'].endswith('.'): + reasons.append('missing trailing .') + except KeyError: + reasons.append('missing target') + return reasons + def __init__(self, value): self.priority = int(value['priority']) self.weight = int(value['weight']) @@ -537,28 +693,30 @@ class SrvRecord(_ValuesMixin, Record): _type = 'SRV' _name_re = re.compile(r'^_[^\.]+\.[^\.]+') - def __init__(self, zone, name, data, source=None): - if not self._name_re.match(name): - raise Exception('Invalid name {}.{}'.format(name, zone.name)) - super(SrvRecord, self).__init__(zone, name, data, source) + @classmethod + def validate(cls, name, data): + reasons = [] + if not cls._name_re.match(name): + reasons.append('invalid name') + reasons.extend(super(SrvRecord, cls).validate(name, data)) + return reasons + + @classmethod + def _validate_value(cls, value): + return SrvValue._validate_value(value) def _process_values(self, values): - ret = [] - for value in values: - try: - ret.append(SrvValue(value)) - except KeyError as e: - raise Exception('Invalid value in record {}, missing {}' - .format(self.fqdn, e.args[0])) - return ret + return [SrvValue(v) for v in values] class TxtRecord(_ValuesMixin, Record): _type = 'TXT' + @classmethod + def _validate_value(cls, value): + if _unescaped_semicolon_re.search(value): + return ['unescaped ;'] + return [] + def _process_values(self, values): - for value in values: - if _unescaped_semicolon_re.search(value): - raise Exception('Invalid record {}, unescaped ;' - .format(self.fqdn)) return values diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 52505cb..99a502e 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -9,7 +9,8 @@ from unittest import TestCase from octodns.record import ARecord, AaaaRecord, AliasRecord, CnameRecord, \ Create, Delete, GeoValue, MxRecord, NaptrRecord, NaptrValue, NsRecord, \ - PtrRecord, Record, SshfpRecord, SpfRecord, SrvRecord, TxtRecord, Update + Record, SshfpRecord, SpfRecord, SrvRecord, TxtRecord, Update, \ + ValidationError from octodns.zone import Zone from helpers import GeoProvider, SimpleProvider @@ -42,15 +43,6 @@ class TestRecord(TestCase): self.assertEquals([b_value], b.values) self.assertEquals(b_data, b.data) - # missing ttl - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, None, {'value': '1.1.1.1'}) - self.assertTrue('missing ttl' in ctx.exception.message) - # missing values & value - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # top-level data = {'ttl': 30, 'value': '4.2.3.4'} self.assertEquals(self.zone.name, ARecord(self.zone, '', data).fqdn) @@ -104,20 +96,6 @@ class TestRecord(TestCase): DummyRecord().__repr__() - def test_invalid_a(self): - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'a', { - 'ttl': 30, - 'values': ['1.2.3.4', 'bar'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def test_geo(self): geo_data = {'ttl': 42, 'values': ['5.2.3.4', '6.2.3.4'], 'geo': {'AF': ['1.1.1.1'], @@ -157,19 +135,6 @@ class TestRecord(TestCase): # Geo provider does consider lack of geo diffs to be changes self.assertTrue(geo.changes(other, geo_target)) - # invalid geo code - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'geo', {'ttl': 42, - 'values': ['5.2.3.4', '6.2.3.4'], - 'geo': {'abc': ['1.1.1.1']}}) - self.assertEquals('Invalid geo "abc"', ctx.exception.message) - - with self.assertRaises(Exception) as ctx: - ARecord(self.zone, 'geo', {'ttl': 42, - 'values': ['5.2.3.4', '6.2.3.4'], - 'geo': {'NA-US': ['1.1.1']}}) - self.assertTrue('not a valid ip' in ctx.exception.message) - # __repr__ doesn't blow up geo.__repr__() @@ -187,30 +152,12 @@ class TestRecord(TestCase): self.assertEquals([b_value], b.values) self.assertEquals(b_data, b.data) - # missing values & value - with self.assertRaises(Exception) as ctx: - _type(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - def test_aaaa(self): a_values = ['2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b', '2001:0db8:3c4d:0015:0000:0000:1a2f:1a3b'] b_value = '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b' self.assertMultipleValues(AaaaRecord, a_values, b_value) - with self.assertRaises(Exception) as ctx: - AaaaRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - AaaaRecord(self.zone, 'a', { - 'ttl': 30, - 'values': [b_value, 'bar'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def assertSingleValue(self, _type, a_value, b_value): a_data = {'ttl': 30, 'value': a_value} a = _type(self.zone, 'a', a_data) @@ -225,11 +172,6 @@ class TestRecord(TestCase): self.assertEquals(b_value, b.value) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - _type(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -251,15 +193,6 @@ class TestRecord(TestCase): self.assertEquals(a_data['value'], a.value) self.assertEquals(a_data, a.data) - # missing value - with self.assertRaises(Exception) as ctx: - AliasRecord(self.zone, None, {'ttl': 0}) - self.assertTrue('missing value' in ctx.exception.message) - # bad name - with self.assertRaises(Exception) as ctx: - AliasRecord(self.zone, None, {'ttl': 0, 'value': 'www.unit.tests'}) - self.assertTrue('missing trailing .' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -277,19 +210,6 @@ class TestRecord(TestCase): self.assertSingleValue(CnameRecord, 'target.foo.com.', 'other.foo.com.') - with self.assertRaises(Exception) as ctx: - CnameRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - CnameRecord(self.zone, 'a', { - 'ttl': 30, - 'values': ['foo.com.', 'bar.com'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def test_mx(self): a_values = [{ 'priority': 10, @@ -319,15 +239,6 @@ class TestRecord(TestCase): self.assertEquals(b_value['value'], b.values[0].value) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - MxRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - MxRecord(self.zone, None, {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -387,15 +298,6 @@ class TestRecord(TestCase): self.assertEquals(b_value[k], getattr(b.values[0], k)) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - NaptrRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - NaptrRecord(self.zone, None, {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -538,33 +440,6 @@ class TestRecord(TestCase): self.assertEquals([b_value], b.values) self.assertEquals(b_data, b.data) - # missing values & value - with self.assertRaises(Exception) as ctx: - NsRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - - with self.assertRaises(Exception) as ctx: - NsRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - with self.assertRaises(Exception) as ctx: - NsRecord(self.zone, 'a', { - 'ttl': 30, - 'values': ['foo.com.', 'bar.com'], - }) - self.assertTrue('Invalid record' in ctx.exception.message) - - def test_ptr(self): - self.assertSingleValue(PtrRecord, 'foo.bar.com.', 'other.bar.com.') - with self.assertRaises(Exception) as ctx: - PtrRecord(self.zone, 'a', { - 'ttl': 30, - 'value': 'foo', - }) - self.assertTrue('Invalid record' in ctx.exception.message) - def test_sshfp(self): a_values = [{ 'algorithm': 10, @@ -599,15 +474,6 @@ class TestRecord(TestCase): self.assertEquals(b_value['fingerprint'], b.values[0].fingerprint) self.assertEquals(b_data, b.data) - # missing value - with self.assertRaises(Exception) as ctx: - SshfpRecord(self.zone, None, {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - SshfpRecord(self.zone, None, {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -677,21 +543,6 @@ class TestRecord(TestCase): self.assertEquals(b_value['target'], b.values[0].target) self.assertEquals(b_data, b.data) - # invalid name - with self.assertRaises(Exception) as ctx: - SrvRecord(self.zone, 'bad', {'ttl': 42}) - self.assertEquals('Invalid name bad.unit.tests.', - ctx.exception.message) - - # missing value - with self.assertRaises(Exception) as ctx: - SrvRecord(self.zone, '_missing._tcp', {'ttl': 42}) - self.assertTrue('missing value(s)' in ctx.exception.message) - # invalid value - with self.assertRaises(Exception) as ctx: - SrvRecord(self.zone, '_missing._udp', {'ttl': 42, 'value': {}}) - self.assertTrue('Invalid value' in ctx.exception.message) - target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) @@ -729,21 +580,6 @@ class TestRecord(TestCase): b_value = 'b other' self.assertMultipleValues(TxtRecord, a_values, b_value) - Record.new(self.zone, 'txt', { - 'ttl': 44, - 'type': 'TXT', - 'value': 'escaped\; foo', - }) - - with self.assertRaises(Exception) as ctx: - Record.new(self.zone, 'txt', { - 'ttl': 44, - 'type': 'TXT', - 'value': 'un-escaped; foo', - }) - self.assertEquals('Invalid record txt.unit.tests., unescaped ;', - ctx.exception.message) - def test_record_new(self): txt = Record.new(self.zone, 'txt', { 'ttl': 44, @@ -794,3 +630,642 @@ class TestRecord(TestCase): self.assertEquals('CA', geo.subdivision_code) self.assertEquals(values, geo.values) self.assertEquals(['NA-US', 'NA'], list(geo.parents)) + + +class TestRecordValidation(TestCase): + zone = Zone('unit.tests.', []) + + def test_base(self): + # no ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'value': '1.2.3.4', + }) + self.assertEquals(['missing ttl'], ctx.exception.reasons) + # invalid ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': -1, + 'value': '1.2.3.4', + }) + self.assertEquals('www.unit.tests.', ctx.exception.fqdn) + self.assertEquals(['invalid ttl'], ctx.exception.reasons) + + def test_A_and_values_mixin(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'values': [ + '1.2.3.4', + '1.2.3.5', + ] + }) + + # missing value(s) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + # missing value(s) & ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + }) + self.assertEquals(['missing ttl', 'missing value(s)'], + ctx.exception.reasons) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'value': 'hello' + }) + self.assertEquals(['invalid ip address "hello"'], + ctx.exception.reasons) + + # invalid ip addresses + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'ttl': 600, + 'values': ['hello', 'goodbye'] + }) + self.assertEquals([ + 'invalid ip address "hello"', + 'invalid ip address "goodbye"' + ], ctx.exception.reasons) + + # invalid & valid ip addresses, no ttl + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'A', + 'values': ['1.2.3.4', 'hello', '5.6.7.8'] + }) + self.assertEquals([ + 'missing ttl', + 'invalid ip address "hello"', + ], ctx.exception.reasons) + + def test_geo(self): + Record.new(self.zone, '', { + 'geo': { + 'NA': ['1.2.3.5'], + 'NA-US': ['1.2.3.5', '1.2.3.6'] + }, + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'geo': { + 'NA': ['hello'], + 'NA-US': ['1.2.3.5', '1.2.3.6'] + }, + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + self.assertEquals(['invalid ip address "hello"'], + ctx.exception.reasons) + + # invalid geo code + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'geo': { + 'XYZ': ['1.2.3.4'], + }, + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + self.assertEquals(['invalid geo "XYZ"'], ctx.exception.reasons) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'geo': { + 'NA': ['hello'], + 'NA-US': ['1.2.3.5', 'goodbye'] + }, + 'type': 'A', + 'ttl': 600, + 'value': '1.2.3.4', + }) + self.assertEquals([ + 'invalid ip address "hello"', + 'invalid ip address "goodbye"' + ], ctx.exception.reasons) + + def test_AAAA(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'value': '2601:644:500:e210:62f8:1dff:feb8:947a', + }) + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'values': [ + '2601:644:500:e210:62f8:1dff:feb8:947a', + '2601:644:500:e210:62f8:1dff:feb8:947b', + ] + }) + + # invalid ip address + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'value': 'hello' + }) + self.assertEquals(['invalid ip address "hello"'], + ctx.exception.reasons) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'value': '1.2.3.4' + }) + self.assertEquals(['invalid ip address "1.2.3.4"'], + ctx.exception.reasons) + + # invalid ip addresses + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'AAAA', + 'ttl': 600, + 'values': ['hello', 'goodbye'] + }) + self.assertEquals([ + 'invalid ip address "hello"', + 'invalid ip address "goodbye"' + ], ctx.exception.reasons) + + def test_ALIAS_and_value_mixin(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'ALIAS', + 'ttl': 600, + 'value': 'foo.bar.com', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_CNAME(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'CNAME', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CNAME', + 'ttl': 600, + 'value': 'foo.bar.com', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_MX(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'priority': 10, + 'value': 'foo.bar.com.' + } + }) + + # missing priority + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'value': 'foo.bar.com.' + } + }) + self.assertEquals(['missing priority'], ctx.exception.reasons) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'priority': 10, + } + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + + def test_NXPTR(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': { + 'order': 10, + 'preference': 20, + 'flags': 'f', + 'service': 'srv', + 'regexp': '.*', + 'replacement': '.' + } + }) + + # missing X priority + value = { + 'order': 10, + 'preference': 20, + 'flags': 'f', + 'service': 'srv', + 'regexp': '.*', + 'replacement': '.' + } + for k in ('order', 'preference', 'flags', 'service', 'regexp', + 'replacement'): + v = dict(value) + del v[k] + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['missing {}'.format(k)], ctx.exception.reasons) + + # non-int order + v = dict(value) + v['order'] = 'boo' + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['invalid order "boo"'], ctx.exception.reasons) + + # non-int preference + v = dict(value) + v['preference'] = 'who' + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['invalid preference "who"'], ctx.exception.reasons) + + def test_NS(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'NS', + 'ttl': 600, + 'values': [ + 'foo.bar.com.', + '1.2.3.4.' + ] + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NS', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # no trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NS', + 'ttl': 600, + 'value': 'foo.bar', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_PTR(self): + # doesn't blow up (name & zone here don't make any sense, but not + # important) + Record.new(self.zone, '', { + 'type': 'PTR', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'PTR', + 'ttl': 600, + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + + # no trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'PTR', + 'ttl': 600, + 'value': 'foo.bar', + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) + + def test_SSHFP(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + + # missing algorithm + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['missing algorithm'], ctx.exception.reasons) + + # invalid algorithm + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 'nope', + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['invalid algorithm "nope"'], ctx.exception.reasons) + + # missing fingerprint_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['missing fingerprint_type'], ctx.exception.reasons) + + # invalid fingerprint_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 'yeeah', + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['invalid fingerprint_type "yeeah"'], + ctx.exception.reasons) + + # missing fingerprint + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 1, + } + }) + self.assertEquals(['missing fingerprint'], ctx.exception.reasons) + + def test_SPF(self): + # doesn't blow up (name & zone here don't make any sense, but not + # important) + Record.new(self.zone, '', { + 'type': 'SPF', + 'ttl': 600, + 'values': [ + 'v=spf1 ip4:192.168.0.1/16-all', + 'v=spf1 ip4:10.1.2.1/24-all', + 'this has some\; semi-colons\; in it', + ] + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SPF', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # missing escapes + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SPF', + 'ttl': 600, + 'value': 'this has some; semi-colons\; in it', + }) + self.assertEquals(['unescaped ;'], ctx.exception.reasons) + + def test_SRV(self): + # doesn't blow up + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + + # invalid name + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'neup', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid name'], ctx.exception.reasons) + + # missing priority + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['missing priority'], ctx.exception.reasons) + + # invalid priority + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 'foo', + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid priority "foo"'], ctx.exception.reasons) + + # missing weight + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['missing weight'], ctx.exception.reasons) + # invalid weight + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 'foo', + 'port': 3, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid weight "foo"'], ctx.exception.reasons) + + # missing port + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['missing port'], ctx.exception.reasons) + # invalid port + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 'foo', + 'target': 'foo.bar.baz.' + } + }) + self.assertEquals(['invalid port "foo"'], ctx.exception.reasons) + + # missing target + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + } + }) + self.assertEquals(['missing target'], ctx.exception.reasons) + # invalid target + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '_srv._tcp', { + 'type': 'SRV', + 'ttl': 600, + 'value': { + 'priority': 1, + 'weight': 2, + 'port': 3, + 'target': 'foo.bar.baz' + } + }) + self.assertEquals(['missing trailing .'], + ctx.exception.reasons) + + def test_TXT(self): + # doesn't blow up (name & zone here don't make any sense, but not + # important) + Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + 'values': [ + 'hello world', + 'this has some\; semi-colons\; in it', + ] + }) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + }) + self.assertEquals(['missing value(s)'], ctx.exception.reasons) + + # missing escapes + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + 'value': 'this has some; semi-colons\; in it', + }) + self.assertEquals(['unescaped ;'], ctx.exception.reasons) From a97818b6ec745a624eeca6e2ace71f93dfc65918 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 09:01:25 -0700 Subject: [PATCH 10/84] populating existing provider state is lenient - adds lenient flag to Record.new, problems during validation are just warnings if it's true - target populate calls during the plan phase pass lenient=True - make all of the provider.populate call logging consistent including both target and lenient - add source=self to Record.new in a few places that were missing it --- octodns/provider/base.py | 2 +- octodns/provider/cloudflare.py | 8 +++++--- octodns/provider/dnsimple.py | 8 +++++--- octodns/provider/dyn.py | 9 ++++++--- octodns/provider/ns1.py | 8 +++++--- octodns/provider/powerdns.py | 7 ++++--- octodns/provider/route53.py | 9 ++++++--- octodns/provider/yaml.py | 9 ++++++--- octodns/record.py | 15 ++++++++++----- octodns/source/base.py | 8 ++++++++ octodns/source/tinydns.py | 19 +++++++++++-------- tests/helpers.py | 4 ++-- tests/test_octodns_provider_base.py | 4 ++-- tests/test_octodns_provider_route53.py | 6 +++--- tests/test_octodns_record.py | 16 ++++++++++++++++ 15 files changed, 90 insertions(+), 42 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 385fe36..2fd4349 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -104,7 +104,7 @@ class BaseProvider(BaseSource): self.log.info('plan: desired=%s', desired.name) existing = Zone(desired.name, desired.sub_zones) - self.populate(existing, target=True) + self.populate(existing, target=True, lenient=True) # compute the changes at the zone/record level changes = existing.changes(desired, self) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index a45bd44..51b2171 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -154,8 +154,9 @@ class CloudflareProvider(BaseProvider): return self._zone_records[zone.name] - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) before = len(zone.records) records = self.zone_records(zone) @@ -171,7 +172,8 @@ class CloudflareProvider(BaseProvider): for _type, records in types.items(): data_for = getattr(self, '_data_for_{}'.format(_type)) data = data_for(_type, records) - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index cb0f2d7..91bd638 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -234,8 +234,9 @@ class DnsimpleProvider(BaseProvider): return self._zone_records[zone.name] - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) values = defaultdict(lambda: defaultdict(list)) for record in self.zone_records(zone): @@ -252,7 +253,8 @@ class DnsimpleProvider(BaseProvider): for name, types in values.items(): for _type, records in types.items(): data_for = getattr(self, '_data_for_{}'.format(_type)) - record = Record.new(zone, name, data_for(_type, records)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 673e8d0..5e0e1f3 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -338,8 +338,10 @@ class DynProvider(BaseProvider): return td_records - def populate(self, zone, target=False): - self.log.info('populate: zone=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + before = len(zone.records) self._check_dyn_sess() @@ -364,7 +366,8 @@ class DynProvider(BaseProvider): for _type, records in types.items(): data_for = getattr(self, '_data_for_{}'.format(_type)) data = data_for(_type, records) - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) if record not in td_records: zone.add_record(record) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index c50341d..33fb19c 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -111,8 +111,9 @@ class Ns1Provider(BaseProvider): 'values': values, } - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) try: nsone_zone = self._client.loadZone(zone.name[:-1]) @@ -127,7 +128,8 @@ class Ns1Provider(BaseProvider): _type = record['type'] data_for = getattr(self, '_data_for_{}'.format(_type)) name = zone.hostname_from_fqdn(record['domain']) - record = Record.new(zone, name, data_for(_type, record)) + record = Record.new(zone, name, data_for(_type, record), + source=self, lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 21e4d44..d8cccae 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -146,8 +146,9 @@ class PowerDnsBaseProvider(BaseProvider): 'ttl': rrset['ttl'] } - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) resp = None try: @@ -177,7 +178,7 @@ class PowerDnsBaseProvider(BaseProvider): data_for = getattr(self, '_data_for_{}'.format(_type)) record_name = zone.hostname_from_fqdn(rrset['name']) record = Record.new(zone, record_name, data_for(rrset), - source=self) + source=self, lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 12c1aff..3875bd6 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -418,8 +418,10 @@ class Route53Provider(BaseProvider): return self._r53_rrsets[zone_id] - def populate(self, zone, target=False): - self.log.debug('populate: name=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + before = len(zone.records) zone_id = self._get_zone_id(zone.name) @@ -449,7 +451,8 @@ class Route53Provider(BaseProvider): data['geo'] = geo else: data = data[0] - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index c728caf..fe1a406 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -45,8 +45,10 @@ class YamlProvider(BaseProvider): self.default_ttl = default_ttl self.enforce_order = enforce_order - def populate(self, zone, target=False): - self.log.debug('populate: zone=%s, target=%s', zone.name, target) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + if target: # When acting as a target we ignore any existing records so that we # create a completely new copy @@ -63,7 +65,8 @@ class YamlProvider(BaseProvider): for d in data: if 'ttl' not in d: d['ttl'] = self.default_ttl - record = Record.new(zone, name, d, source=self) + record = Record.new(zone, name, d, source=self, + lenient=lenient) zone.add_record(record) self.log.info('populate: found %s records', diff --git a/octodns/record.py b/octodns/record.py index 827ad0a..3f07c39 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -56,10 +56,12 @@ class Delete(Change): class ValidationError(Exception): + @classmethod + def build_message(cls, fqdn, reasons): + return 'Invalid record {}\n - {}'.format(fqdn, '\n - '.join(reasons)) + def __init__(self, fqdn, reasons): - message = 'Invalid record {}\n - {}' \ - .format(fqdn, '\n - '.join(reasons)) - super(Exception, self).__init__(message) + super(Exception, self).__init__(self.build_message(fqdn, reasons)) self.fqdn = fqdn self.reasons = reasons @@ -68,7 +70,7 @@ class Record(object): log = getLogger('Record') @classmethod - def new(cls, zone, name, data, source=None): + def new(cls, zone, name, data, source=None, lenient=False): fqdn = '{}.{}'.format(name, zone.name) if name else zone.name try: _type = data['type'] @@ -107,7 +109,10 @@ class Record(object): raise Exception('Unknown record type: "{}"'.format(_type)) reasons = _class.validate(name, data) if reasons: - raise ValidationError(fqdn, reasons) + if lenient: + cls.log.warn(ValidationError.build_message(fqdn, reasons)) + else: + raise ValidationError(fqdn, reasons) return _class(zone, name, data, source=source) @classmethod diff --git a/octodns/source/base.py b/octodns/source/base.py index 42d214b..2e2c5c2 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -23,6 +23,14 @@ class BaseSource(object): def populate(self, zone, target=False): ''' Loads all zones the provider knows about + + When `target` is True the populate call is being made to load the + current state of the provider. + + When `lenient` is True the populate call may skip record validation and + do a "best effort" load of data. That will allow through some common, + but not best practices stuff that we otherwise would reject. E.g. no + trailing . or mising escapes for ;. ''' raise NotImplementedError('Abstract base class, populate method ' 'missing') diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 6805378..1b98092 100644 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -81,19 +81,21 @@ class TinyDnsBaseSource(BaseSource): 'values': ['{}.'.format(r[0]) for r in records] } - def populate(self, zone, target=False): - self.log.debug('populate: zone=%s', zone.name) + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + before = len(zone.records) if zone.name.endswith('in-addr.arpa.'): - self._populate_in_addr_arpa(zone) + self._populate_in_addr_arpa(zone, lenient) else: - self._populate_normal(zone) + self._populate_normal(zone, lenient) self.log.info('populate: found %s records', len(zone.records) - before) - def _populate_normal(self, zone): + def _populate_normal(self, zone, lenient): type_map = { '=': 'A', '^': None, @@ -129,14 +131,15 @@ class TinyDnsBaseSource(BaseSource): data_for = getattr(self, '_data_for_{}'.format(_type)) data = data_for(_type, d) if data: - record = Record.new(zone, name, data, source=self) + record = Record.new(zone, name, data, source=self, + lenient=lenient) try: zone.add_record(record) except SubzoneRecordException: self.log.debug('_populate_normal: skipping subzone ' 'record=%s', record) - def _populate_in_addr_arpa(self, zone): + def _populate_in_addr_arpa(self, zone, lenient): name_re = re.compile('(?P.+)\.{}$'.format(zone.name[:-1])) for line in self._lines(): @@ -170,7 +173,7 @@ class TinyDnsBaseSource(BaseSource): 'ttl': ttl, 'type': 'PTR', 'value': value - }, source=self) + }, source=self, lenient=lenient) try: zone.add_record(record) except DuplicateRecordException: diff --git a/tests/helpers.py b/tests/helpers.py index a8aafa3..adac81d 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -22,7 +22,7 @@ class SimpleProvider(object): def __init__(self, id='test'): pass - def populate(self, zone, source=True): + def populate(self, zone, source=False, lenient=False): pass def supports(self, record): @@ -38,7 +38,7 @@ class GeoProvider(object): def __init__(self, id='test'): pass - def populate(self, zone, source=True): + def populate(self, zone, source=False, lenient=False): pass def supports(self, record): diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 766bf65..bd134bc 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -24,7 +24,7 @@ class HelperProvider(BaseProvider): self.apply_disabled = apply_disabled self.include_change_callback = include_change_callback - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): pass def _include_change(self, change): @@ -72,7 +72,7 @@ class TestBaseProvider(TestCase): class HasPopulate(HasSupports): - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): zone.add_record(Record.new(zone, '', { 'ttl': 60, 'type': 'A', diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 5982b74..0a769f9 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -370,7 +370,7 @@ class TestRoute53Provider(TestCase): stubber.assert_no_pending_responses() # Delete by monkey patching in a populate that includes an extra record - def add_extra_populate(existing, target): + def add_extra_populate(existing, target, lenient): for record in self.expected.records: existing.records.add(record) record = Record.new(existing, 'extra', @@ -406,7 +406,7 @@ class TestRoute53Provider(TestCase): # Update by monkey patching in a populate that modifies the A record # with geos - def mod_geo_populate(existing, target): + def mod_geo_populate(existing, target, lenient): for record in self.expected.records: if record._type != 'A' or not record.geo: existing.records.add(record) @@ -502,7 +502,7 @@ class TestRoute53Provider(TestCase): # Update converting to non-geo by monkey patching in a populate that # modifies the A record with geos - def mod_add_geo_populate(existing, target): + def mod_add_geo_populate(existing, target, lenient): for record in self.expected.records: if record._type != 'A' or record.geo: existing.records.add(record) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 99a502e..6e40e18 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -643,6 +643,7 @@ class TestRecordValidation(TestCase): 'value': '1.2.3.4', }) self.assertEquals(['missing ttl'], ctx.exception.reasons) + # invalid ttl with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, 'www', { @@ -653,6 +654,21 @@ class TestRecordValidation(TestCase): self.assertEquals('www.unit.tests.', ctx.exception.fqdn) self.assertEquals(['invalid ttl'], ctx.exception.reasons) + # no exception if we're in lenient mode + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': -1, + 'value': '1.2.3.4', + }, lenient=True) + + # __init__ may still blow up, even if validation is lenient + with self.assertRaises(KeyError) as ctx: + Record.new(self.zone, 'www', { + 'type': 'A', + 'ttl': -1, + }, lenient=True) + self.assertEquals(('value',), ctx.exception.args) + def test_A_and_values_mixin(self): # doesn't blow up Record.new(self.zone, '', { From cfc0d586a13450ccf60ddc8b8fab260da958442b Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 09:06:21 -0700 Subject: [PATCH 11/84] Log max_workers, useful to know --- octodns/manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octodns/manager.py b/octodns/manager.py index 0366685..5feed8e 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -69,6 +69,7 @@ class Manager(object): manager_config = self.config.get('manager', {}) max_workers = manager_config.get('max_workers', 1) \ if max_workers is None else max_workers + self.log.info('__init__: max_workers=%d', max_workers) if max_workers > 1: self._executor = ThreadPoolExecutor(max_workers=max_workers) else: From a69ff64ae1db0b19eed101e1c992fc965e06b24a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 09:24:25 -0700 Subject: [PATCH 12/84] Add --lenient flag to dump --- octodns/cmds/dump.py | 5 ++++- octodns/manager.py | 4 ++-- tests/test_octodns_manager.py | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/octodns/cmds/dump.py b/octodns/cmds/dump.py index e4c4987..9f5e0aa 100755 --- a/octodns/cmds/dump.py +++ b/octodns/cmds/dump.py @@ -18,6 +18,9 @@ def main(): parser.add_argument('--output-dir', required=True, help='The directory into which the results will be ' 'written (Note: will overwrite existing files)') + parser.add_argument('--lenient', action='store_true', default=False, + help='Ignore record validations and do a best effort ' + 'dump') parser.add_argument('zone', help='Zone to dump') parser.add_argument('source', nargs='+', help='Source(s) to pull data from') @@ -25,7 +28,7 @@ def main(): args = parser.parse_args() manager = Manager(args.config_file) - manager.dump(args.zone, args.output_dir, *args.source) + manager.dump(args.zone, args.output_dir, args.lenient, *args.source) if __name__ == '__main__': diff --git a/octodns/manager.py b/octodns/manager.py index 5feed8e..07719dd 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -323,7 +323,7 @@ class Manager(object): return zb.changes(za, _AggregateTarget(a + b)) - def dump(self, zone, output_dir, source, *sources): + def dump(self, zone, output_dir, lenient, source, *sources): ''' Dump zone data from the specified source ''' @@ -342,7 +342,7 @@ class Manager(object): zone = Zone(zone, self.configured_sub_zones(zone)) for source in sources: - source.populate(zone) + source.populate(zone, lenient=lenient) plan = target.plan(zone) target.apply(plan) diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index fa8bdd1..641c1ff 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -195,15 +195,15 @@ class TestManager(TestCase): manager = Manager(get_config_filename('simple.yaml')) with self.assertRaises(Exception) as ctx: - manager.dump('unit.tests.', tmpdir.dirname, 'nope') + manager.dump('unit.tests.', tmpdir.dirname, False, 'nope') self.assertEquals('Unknown source: nope', ctx.exception.message) - manager.dump('unit.tests.', tmpdir.dirname, 'in') + manager.dump('unit.tests.', tmpdir.dirname, False, 'in') # make sure this fails with an IOError and not a KeyError when # tyring to find sub zones with self.assertRaises(IOError): - manager.dump('unknown.zone.', tmpdir.dirname, 'in') + manager.dump('unknown.zone.', tmpdir.dirname, False, 'in') def test_validate_configs(self): Manager(get_config_filename('simple-validate.yaml')).validate_configs() From d2af8efe5c9d7b5f19bbb5ed6dacb52caac4e172 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 09:49:11 -0700 Subject: [PATCH 13/84] Root CNAMEs are not allowed --- octodns/record.py | 8 ++++++++ tests/test_octodns_provider_route53.py | 6 ++++-- tests/test_octodns_record.py | 13 +++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 3f07c39..11876f5 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -400,6 +400,14 @@ class AliasRecord(_ValueMixin, Record): class CnameRecord(_ValueMixin, Record): _type = 'CNAME' + @classmethod + def validate(cls, name, data): + reasons = [] + if name == '': + reasons.append('root CNAME not allowed') + reasons.extend(super(CnameRecord, cls).validate(name, data)) + return reasons + @classmethod def _validate_value(cls, value): reasons = [] diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 0a769f9..8960088 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1260,8 +1260,10 @@ class TestRoute53Records(TestCase): False) self.assertEquals(c, c) d = _Route53Record(None, Record.new(existing, '', - {'ttl': 42, 'type': 'CNAME', - 'value': 'foo.bar.'}), + {'ttl': 42, 'type': 'MX', + 'value': { + 'priority': 10, + 'value': 'foo.bar.'}}), False) self.assertEquals(d, d) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 6e40e18..96a83a0 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -859,15 +859,24 @@ class TestRecordValidation(TestCase): def test_CNAME(self): # doesn't blow up - Record.new(self.zone, '', { + Record.new(self.zone, 'www', { 'type': 'CNAME', 'ttl': 600, 'value': 'foo.bar.com.', }) - # missing trailing . + # root cname is a no-no with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { + 'type': 'CNAME', + 'ttl': 600, + 'value': 'foo.bar.com.', + }) + self.assertEquals(['root CNAME not allowed'], ctx.exception.reasons) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, 'www', { 'type': 'CNAME', 'ttl': 600, 'value': 'foo.bar.com', From 615bc95976ca0dba4ce8d77b06d19205d1fe7a08 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 09:49:25 -0700 Subject: [PATCH 14/84] CNAME cannot coexist with other records on a node --- octodns/zone.py | 18 +++++++++++++++--- tests/test_octodns_zone.py | 27 ++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/octodns/zone.py b/octodns/zone.py index 1822fec..03bc41c 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -19,6 +19,10 @@ class DuplicateRecordException(Exception): pass +class InvalidNodeException(Exception): + pass + + def _is_eligible(record): # Should this record be considered when computing changes # We ignore all top-level NS records @@ -59,9 +63,17 @@ class Zone(object): raise SubzoneRecordException('Record {} a managed sub-zone ' 'and not of type NS' .format(record.fqdn)) - if record in self.records: - raise DuplicateRecordException('Duplicate record {}, type {}' - .format(record.fqdn, record._type)) + for existing in self.records: + if record == existing: + raise DuplicateRecordException('Duplicate record {}, type {}' + .format(record.fqdn, + record._type)) + elif name == existing.name and (record._type == 'CNAME' or + existing._type == 'CNAME'): + raise InvalidNodeException('Invalid state, CNAME at {} ' + 'cannot coexist with other records' + .format(record.fqdn)) + self.records.add(record) def changes(self, desired, target): diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index 88bbb68..a4d7300 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -8,7 +8,8 @@ from __future__ import absolute_import, division, print_function, \ from unittest import TestCase from octodns.record import ARecord, AaaaRecord, Create, Delete, Record, Update -from octodns.zone import DuplicateRecordException, SubzoneRecordException, Zone +from octodns.zone import DuplicateRecordException, InvalidNodeException, \ + SubzoneRecordException, Zone from helpers import SimpleProvider @@ -205,3 +206,27 @@ class TestZone(TestCase): self.assertTrue(zone_missing.changes(zone_normal, provider)) self.assertFalse(zone_missing.changes(zone_ignored, provider)) + + def test_cname_coexisting(self): + zone = Zone('unit.tests.', []) + a = Record.new(zone, 'www', { + 'ttl': 60, + 'type': 'A', + 'value': '9.9.9.9', + }) + cname = Record.new(zone, 'www', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'foo.bar.com.', + }) + + # add cname to a + zone.add_record(a) + with self.assertRaises(InvalidNodeException): + zone.add_record(cname) + + # add a to cname + zone = Zone('unit.tests.', []) + zone.add_record(cname) + with self.assertRaises(InvalidNodeException): + zone.add_record(a) From 1340aee8a934c690d74001b3780880b5eee8b2fa Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 13:04:38 -0700 Subject: [PATCH 15/84] MX RFC1035 - priority -> preference & value -> exchange --- octodns/provider/cloudflare.py | 8 ++-- octodns/provider/dnsimple.py | 8 ++-- octodns/provider/dyn.py | 6 +-- octodns/provider/ns1.py | 8 ++-- octodns/provider/powerdns.py | 8 ++-- octodns/provider/route53.py | 9 ++-- octodns/record.py | 41 +++++++++++------- octodns/source/tinydns.py | 4 +- tests/config/unit.tests.yaml | 12 +++--- tests/test_octodns_provider_dyn.py | 8 ++-- tests/test_octodns_provider_ns1.py | 8 ++-- tests/test_octodns_provider_route53.py | 12 +++--- tests/test_octodns_record.py | 58 ++++++++++++++++---------- tests/test_octodns_source_tinydns.py | 16 +++---- 14 files changed, 118 insertions(+), 88 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 51b2171..2ee8f8b 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -116,8 +116,8 @@ class CloudflareProvider(BaseProvider): values = [] for r in records: values.append({ - 'priority': r['priority'], - 'value': '{}.'.format(r['content']), + 'preference': r['priority'], + 'exchange': '{}.'.format(r['content']), }) return { 'ttl': records[0]['ttl'], @@ -207,8 +207,8 @@ class CloudflareProvider(BaseProvider): def _contents_for_MX(self, record): for value in record.values: yield { - 'priority': value.priority, - 'content': value.value + 'priority': value.preference, + 'content': value.exchange } def _apply_Create(self, change): diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index 91bd638..dc44d1b 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -128,8 +128,8 @@ class DnsimpleProvider(BaseProvider): values = [] for record in records: values.append({ - 'priority': record['priority'], - 'value': '{}.'.format(record['content']) + 'preference': record['priority'], + 'exchange': '{}.'.format(record['content']) }) return { 'ttl': records[0]['ttl'], @@ -290,9 +290,9 @@ class DnsimpleProvider(BaseProvider): def _params_for_MX(self, record): for value in record.values: yield { - 'content': value.value, + 'content': value.exchange, 'name': record.name, - 'priority': value.priority, + 'priority': value.preference, 'ttl': record.ttl, 'type': record._type } diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 5e0e1f3..e21b93e 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -206,7 +206,7 @@ class DynProvider(BaseProvider): return { 'type': _type, 'ttl': records[0].ttl, - 'values': [{'priority': r.preference, 'value': r.exchange} + 'values': [{'preference': r.preference, 'exchange': r.exchange} for r in records], } @@ -400,8 +400,8 @@ class DynProvider(BaseProvider): def _kwargs_for_MX(self, record): return [{ - 'preference': v.priority, - 'exchange': v.value, + 'preference': v.preference, + 'exchange': v.exchange, 'ttl': record.ttl, } for v in record.values] diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 33fb19c..2f0a024 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -57,10 +57,10 @@ class Ns1Provider(BaseProvider): def _data_for_MX(self, _type, record): values = [] for answer in record['short_answers']: - priority, value = answer.split(' ', 1) + preference, exchange = answer.split(' ', 1) values.append({ - 'priority': priority, - 'value': value, + 'preference': preference, + 'exchange': exchange, }) return { 'ttl': record['ttl'], @@ -150,7 +150,7 @@ class Ns1Provider(BaseProvider): _params_for_PTR = _params_for_CNAME def _params_for_MX(self, record): - values = [(v.priority, v.value) for v in record.values] + values = [(v.preference, v.exchange) for v in record.values] return {'answers': values, 'ttl': record.ttl} def _params_for_NAPTR(self, record): diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index d8cccae..c6d11b0 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -83,10 +83,10 @@ class PowerDnsBaseProvider(BaseProvider): def _data_for_MX(self, rrset): values = [] for record in rrset['records']: - priority, value = record['content'].split(' ', 1) + preference, exchange = record['content'].split(' ', 1) values.append({ - 'priority': priority, - 'value': value, + 'preference': preference, + 'exchange': exchange, }) return { 'type': rrset['type'], @@ -208,7 +208,7 @@ class PowerDnsBaseProvider(BaseProvider): def _records_for_MX(self, record): return [{ - 'content': '{} {}'.format(v.priority, v.value), + 'content': '{} {}'.format(v.preference, v.exchange), 'disabled': False } for v in record.values] diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 3875bd6..b4fcab8 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -96,7 +96,8 @@ class _Route53Record(object): _values_for_PTR = _values_for_value def _values_for_MX(self, record): - return ['{} {}'.format(v.priority, v.value) for v in record.values] + return ['{} {}'.format(v.preference, v.exchange) + for v in record.values] def _values_for_NAPTR(self, record): return ['{} {} "{}" "{}" "{}" {}' @@ -335,10 +336,10 @@ class Route53Provider(BaseProvider): def _data_for_MX(self, rrset): values = [] for rr in rrset['ResourceRecords']: - priority, value = rr['Value'].split(' ') + preference, exchange = rr['Value'].split(' ') values.append({ - 'priority': priority, - 'value': value, + 'preference': preference, + 'exchange': exchange, }) return { 'type': rrset['Type'], diff --git a/octodns/record.py b/octodns/record.py index 11876f5..0388911 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -424,32 +424,45 @@ class MxValue(object): @classmethod def _validate_value(cls, value): reasons = [] - if 'priority' not in value: - reasons.append('missing priority') - if 'value' not in value: - reasons.append('missing value') + if 'preference' not in value and 'priority' not in value: + reasons.append('missing preference') + exchange = None + try: + exchange = value.get('exchange', None) or value['value'] + if not exchange.endswith('.'): + reasons.append('missing trailing .') + except KeyError: + reasons.append('missing exchange') return reasons def __init__(self, value): - # TODO: rename preference - self.priority = int(value['priority']) - # TODO: rename to exchange? - self.value = value['value'].lower() + # RFC1035 says preference, half the providers use priority + try: + preference = value['preference'] + except KeyError: + preference = value['priority'] + self.preference = int(preference) + # UNTIL 1.0 remove value fallback + try: + exchange = value['exchange'] + except KeyError: + exchange = value['value'] + self.exchange = exchange @property def data(self): return { - 'priority': self.priority, - 'value': self.value, + 'preference': self.preference, + 'exchange': self.exchange, } def __cmp__(self, other): - if self.priority == other.priority: - return cmp(self.value, other.value) - return cmp(self.priority, other.priority) + if self.preference == other.preference: + return cmp(self.exchange, other.exchange) + return cmp(self.preference, other.preference) def __repr__(self): - return "'{} {}'".format(self.priority, self.value) + return "'{} {}'".format(self.preference, self.exchange) class MxRecord(_ValuesMixin, Record): diff --git a/octodns/source/tinydns.py b/octodns/source/tinydns.py index 1b98092..63cafa2 100644 --- a/octodns/source/tinydns.py +++ b/octodns/source/tinydns.py @@ -65,8 +65,8 @@ class TinyDnsBaseSource(BaseSource): 'ttl': ttl, 'type': _type, 'values': [{ - 'priority': r[1], - 'value': '{}.'.format(r[0]) + 'preference': r[1], + 'exchange': '{}.'.format(r[0]) } for r in records] } diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index d18bf59..8be1614 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -60,12 +60,12 @@ mx: ttl: 300 type: MX values: - - priority: 40 - value: smtp-1.unit.tests. - - priority: 20 - value: smtp-2.unit.tests. - - priority: 30 - value: smtp-3.unit.tests. + - exchange: smtp-1.unit.tests. + preference: 40 + - exchange: smtp-2.unit.tests. + preference: 20 + - exchange: smtp-3.unit.tests. + preference: 30 - priority: 10 value: smtp-4.unit.tests. naptr: diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 307e640..bebd3e3 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -46,11 +46,11 @@ class TestDynProvider(TestCase): 'type': 'MX', 'ttl': 302, 'values': [{ - 'priority': 10, - 'value': 'smtp-1.unit.tests.' + 'preference': 10, + 'exchange': 'smtp-1.unit.tests.' }, { - 'priority': 20, - 'value': 'smtp-2.unit.tests.' + 'preference': 20, + 'exchange': 'smtp-2.unit.tests.' }] }), ('naptr', { diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index acb0125..ecc107c 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -44,11 +44,11 @@ class TestNs1Provider(TestCase): 'ttl': 35, 'type': 'MX', 'values': [{ - 'priority': 10, - 'value': 'mx1.unit.tests.', + 'preference': 10, + 'exchange': 'mx1.unit.tests.', }, { - 'priority': 20, - 'value': 'mx2.unit.tests.', + 'preference': 20, + 'exchange': 'mx2.unit.tests.', }] })) expected.add(Record.new(zone, 'naptr', { diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 8960088..cad58f8 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -52,11 +52,11 @@ class TestRoute53Provider(TestCase): 'Goodbye World?']}), ('', {'ttl': 64, 'type': 'MX', 'values': [{ - 'priority': 10, - 'value': 'smtp-1.unit.tests.', + 'preference': 10, + 'exchange': 'smtp-1.unit.tests.', }, { - 'priority': 20, - 'value': 'smtp-2.unit.tests.', + 'preference': 20, + 'exchange': 'smtp-2.unit.tests.', }]}), ('naptr', {'ttl': 65, 'type': 'NAPTR', 'value': { @@ -1262,8 +1262,8 @@ class TestRoute53Records(TestCase): d = _Route53Record(None, Record.new(existing, '', {'ttl': 42, 'type': 'MX', 'value': { - 'priority': 10, - 'value': 'foo.bar.'}}), + 'preference': 10, + 'exchange': 'foo.bar.'}}), False) self.assertEquals(d, d) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 96a83a0..cb87c70 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -212,45 +212,49 @@ class TestRecord(TestCase): def test_mx(self): a_values = [{ - 'priority': 10, - 'value': 'smtp1' + 'preference': 10, + 'exchange': 'smtp1.' }, { 'priority': 20, - 'value': 'smtp2' + 'value': 'smtp2.' }] a_data = {'ttl': 30, 'values': a_values} a = MxRecord(self.zone, 'a', a_data) self.assertEquals('a', a.name) self.assertEquals('a.unit.tests.', a.fqdn) self.assertEquals(30, a.ttl) - self.assertEquals(a_values[0]['priority'], a.values[0].priority) - self.assertEquals(a_values[0]['value'], a.values[0].value) - self.assertEquals(a_values[1]['priority'], a.values[1].priority) - self.assertEquals(a_values[1]['value'], a.values[1].value) + self.assertEquals(a_values[0]['preference'], a.values[0].preference) + self.assertEquals(a_values[0]['exchange'], a.values[0].exchange) + self.assertEquals(a_values[1]['priority'], a.values[1].preference) + self.assertEquals(a_values[1]['value'], a.values[1].exchange) + a_data['values'][1] = { + 'preference': 20, + 'exchange': 'smtp2.', + } self.assertEquals(a_data, a.data) b_value = { - 'priority': 12, - 'value': 'smtp3', + 'preference': 12, + 'exchange': 'smtp3.', } b_data = {'ttl': 30, 'value': b_value} b = MxRecord(self.zone, 'b', b_data) - self.assertEquals(b_value['priority'], b.values[0].priority) - self.assertEquals(b_value['value'], b.values[0].value) + self.assertEquals(b_value['preference'], b.values[0].preference) + self.assertEquals(b_value['exchange'], b.values[0].exchange) self.assertEquals(b_data, b.data) target = SimpleProvider() # No changes with self self.assertFalse(a.changes(a, target)) - # Diff in priority causes change + # Diff in preference causes change other = MxRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) - other.values[0].priority = 22 + other.values[0].preference = 22 change = a.changes(other, target) self.assertEqual(change.existing, a) self.assertEqual(change.new, other) # Diff in value causes change - other.values[0].priority = a.values[0].priority - other.values[0].value = 'smtpX' + other.values[0].preference = a.values[0].preference + other.values[0].exchange = 'smtpX' change = a.changes(other, target) self.assertEqual(change.existing, a) self.assertEqual(change.new, other) @@ -889,8 +893,8 @@ class TestRecordValidation(TestCase): 'type': 'MX', 'ttl': 600, 'value': { - 'priority': 10, - 'value': 'foo.bar.com.' + 'preference': 10, + 'exchange': 'foo.bar.com.' } }) @@ -900,10 +904,10 @@ class TestRecordValidation(TestCase): 'type': 'MX', 'ttl': 600, 'value': { - 'value': 'foo.bar.com.' + 'exchange': 'foo.bar.com.' } }) - self.assertEquals(['missing priority'], ctx.exception.reasons) + self.assertEquals(['missing preference'], ctx.exception.reasons) # missing value with self.assertRaises(ValidationError) as ctx: @@ -911,10 +915,22 @@ class TestRecordValidation(TestCase): 'type': 'MX', 'ttl': 600, 'value': { - 'priority': 10, + 'preference': 10, } }) - self.assertEquals(['missing value'], ctx.exception.reasons) + self.assertEquals(['missing exchange'], ctx.exception.reasons) + + # missing trailing . + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'preference': 10, + 'exchange': 'foo.bar.com' + } + }) + self.assertEquals(['missing trailing .'], ctx.exception.reasons) def test_NXPTR(self): # doesn't blow up diff --git a/tests/test_octodns_source_tinydns.py b/tests/test_octodns_source_tinydns.py index b4cea06..5792b25 100644 --- a/tests/test_octodns_source_tinydns.py +++ b/tests/test_octodns_source_tinydns.py @@ -68,22 +68,22 @@ class TestTinyDnsFileSource(TestCase): 'type': 'MX', 'ttl': 3600, 'values': [{ - 'priority': 10, - 'value': 'smtp-1-host.example.com.', + 'preference': 10, + 'exchange': 'smtp-1-host.example.com.', }, { - 'priority': 20, - 'value': 'smtp-2-host.example.com.', + 'preference': 20, + 'exchange': 'smtp-2-host.example.com.', }] }), ('smtp', { 'type': 'MX', 'ttl': 1800, 'values': [{ - 'priority': 30, - 'value': 'smtp-1-host.example.com.', + 'preference': 30, + 'exchange': 'smtp-1-host.example.com.', }, { - 'priority': 40, - 'value': 'smtp-2-host.example.com.', + 'preference': 40, + 'exchange': 'smtp-2-host.example.com.', }] }), ): From 6fc82fd279e32c9e1f35d98a8d32575ba49831a3 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 13:17:32 -0700 Subject: [PATCH 16/84] Validate that MX preference parses as int --- octodns/record.py | 8 +++++++- tests/test_octodns_record.py | 16 ++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 0388911..03ce675 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -424,8 +424,14 @@ class MxValue(object): @classmethod def _validate_value(cls, value): reasons = [] - if 'preference' not in value and 'priority' not in value: + try: + # seperate lines to have preference set in the ValueError case + preference = value.get('preference', None) or value['priority'] + int(preference) + except KeyError: reasons.append('missing preference') + except ValueError: + reasons.append('invalid preference "{}"'.format(preference)) exchange = None try: exchange = value.get('exchange', None) or value['value'] diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index cb87c70..0230e2c 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -898,7 +898,7 @@ class TestRecordValidation(TestCase): } }) - # missing priority + # missing preference with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'MX', @@ -909,7 +909,19 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['missing preference'], ctx.exception.reasons) - # missing value + # invalid preference + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'MX', + 'ttl': 600, + 'value': { + 'preference': 'nope', + 'exchange': 'foo.bar.com.' + } + }) + self.assertEquals(['invalid preference "nope"'], ctx.exception.reasons) + + # missing exchange with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'MX', From 3ce0d71e62f95aa86cf698435851176905f8a194 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 13:28:22 -0700 Subject: [PATCH 17/84] NAPTR RFC2915 - validate flags (partial) - punting on service, regex & replacement validation for now - clean up MX a smidge --- octodns/record.py | 19 +++++++++++++------ tests/test_octodns_record.py | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 03ce675..1d2b349 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -425,13 +425,12 @@ class MxValue(object): def _validate_value(cls, value): reasons = [] try: - # seperate lines to have preference set in the ValueError case - preference = value.get('preference', None) or value['priority'] - int(preference) + int(value.get('preference', None) or value['priority']) except KeyError: reasons.append('missing preference') except ValueError: - reasons.append('invalid preference "{}"'.format(preference)) + reasons.append('invalid preference "{}"' + .format(value['preference'])) exchange = None try: exchange = value.get('exchange', None) or value['value'] @@ -483,6 +482,7 @@ class MxRecord(_ValuesMixin, Record): class NaptrValue(object): + LEGAL_FLAGS = ('S', 'A', 'U', 'P') @classmethod def _validate_value(cls, data): @@ -500,8 +500,15 @@ class NaptrValue(object): except ValueError: reasons.append('invalid preference "{}"' .format(data['preference'])) - # TODO: validate field data - for k in ('flags', 'service', 'regexp', 'replacement'): + try: + flags = data['flags'] + if flags not in cls.LEGAL_FLAGS: + reasons.append('invalid flags "{}"'.format(flags)) + except KeyError: + reasons.append('missing flags') + + # TODO: validate these... they're non-trivial + for k in ('service', 'regexp', 'replacement'): if k not in data: reasons.append('missing {}'.format(k)) return reasons diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 0230e2c..08d3ad7 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -952,7 +952,7 @@ class TestRecordValidation(TestCase): 'value': { 'order': 10, 'preference': 20, - 'flags': 'f', + 'flags': 'S', 'service': 'srv', 'regexp': '.*', 'replacement': '.' @@ -963,7 +963,7 @@ class TestRecordValidation(TestCase): value = { 'order': 10, 'preference': 20, - 'flags': 'f', + 'flags': 'S', 'service': 'srv', 'regexp': '.*', 'replacement': '.' @@ -1002,6 +1002,17 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['invalid preference "who"'], ctx.exception.reasons) + # unrecognized flags + v = dict(value) + v['flags'] = 'X' + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'NAPTR', + 'ttl': 600, + 'value': v + }) + self.assertEquals(['invalid flags "X"'], ctx.exception.reasons) + def test_NS(self): # doesn't blow up Record.new(self.zone, '', { From 4e3cc6b46ac7e324ff028e4056ee4e04f4f0c243 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 23 Jun 2017 13:35:04 -0700 Subject: [PATCH 18/84] SSHFP RFC4255 - validate algorithm & fingerprint_type - unrecognized wording for invalid values --- octodns/record.py | 18 ++++++++++++------ tests/test_octodns_record.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 1d2b349..23bfd45 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -482,7 +482,7 @@ class MxRecord(_ValuesMixin, Record): class NaptrValue(object): - LEGAL_FLAGS = ('S', 'A', 'U', 'P') + VALID_FLAGS = ('S', 'A', 'U', 'P') @classmethod def _validate_value(cls, data): @@ -502,8 +502,8 @@ class NaptrValue(object): .format(data['preference'])) try: flags = data['flags'] - if flags not in cls.LEGAL_FLAGS: - reasons.append('invalid flags "{}"'.format(flags)) + if flags not in cls.VALID_FLAGS: + reasons.append('unrecognized flags "{}"'.format(flags)) except KeyError: reasons.append('missing flags') @@ -594,19 +594,25 @@ class PtrRecord(_ValueMixin, Record): class SshfpValue(object): + VALID_ALGORITHMS = (1, 2) + VALID_FINGERPRINT_TYPES = (1,) @classmethod def _validate_value(cls, value): reasons = [] - # TODO: validate algorithm and fingerprint_type values try: - int(value['algorithm']) + algorithm = int(value['algorithm']) + if algorithm not in cls.VALID_ALGORITHMS: + reasons.append('unrecognized algorithm "{}"'.format(algorithm)) except KeyError: reasons.append('missing algorithm') except ValueError: reasons.append('invalid algorithm "{}"'.format(value['algorithm'])) try: - int(value['fingerprint_type']) + fingerprint_type = int(value['fingerprint_type']) + if fingerprint_type not in cls.VALID_FINGERPRINT_TYPES: + reasons.append('unrecognized fingerprint_type "{}"' + .format(fingerprint_type)) except KeyError: reasons.append('missing fingerprint_type') except ValueError: diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 08d3ad7..1d64081 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -1011,7 +1011,7 @@ class TestRecordValidation(TestCase): 'ttl': 600, 'value': v }) - self.assertEquals(['invalid flags "X"'], ctx.exception.reasons) + self.assertEquals(['unrecognized flags "X"'], ctx.exception.reasons) def test_NS(self): # doesn't blow up @@ -1104,6 +1104,20 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['invalid algorithm "nope"'], ctx.exception.reasons) + # unrecognized algorithm + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 42, + 'fingerprint_type': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['unrecognized algorithm "42"'], + ctx.exception.reasons) + # missing fingerprint_type with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { @@ -1130,6 +1144,20 @@ class TestRecordValidation(TestCase): self.assertEquals(['invalid fingerprint_type "yeeah"'], ctx.exception.reasons) + # unrecognized fingerprint_type + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'SSHFP', + 'ttl': 600, + 'value': { + 'algorithm': 1, + 'fingerprint_type': 42, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73' + } + }) + self.assertEquals(['unrecognized fingerprint_type "42"'], + ctx.exception.reasons) + # missing fingerprint with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { From 598acc943d48eca9ce0c461137d7dfeb242e9d7f Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 23 Jun 2017 15:19:27 -0700 Subject: [PATCH 19/84] Added full support of Azure DNS. TODO: testing. --- .gitignore | 5 +- doit.txt | 4 + octodns/provider/azuredns.py | 297 ++++++++++++++++++----------------- 3 files changed, 164 insertions(+), 142 deletions(-) create mode 100755 doit.txt diff --git a/.gitignore b/.gitignore index eca95c9..5a3f3f9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ nosetests.xml octodns.egg-info/ output/ tmp/ -Makefile build/ -config/ \ No newline at end of file +config/ +./rb.txt +./doit.txt diff --git a/doit.txt b/doit.txt new file mode 100755 index 0000000..1ff2da6 --- /dev/null +++ b/doit.txt @@ -0,0 +1,4 @@ +#!/bin/bash +#script to rebuild octodns quickly + +octodns-sync --config-file=./config/production.yaml --doit diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 66ca8e3..07b39fe 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -8,54 +8,66 @@ import sys from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient -from azure.mgmt.dns.models import * +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone + +from functools import reduce -from collections import defaultdict -# from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. import logging import re from ..record import Record, Update from .base import BaseProvider - -#TODO: changes made to master include adding /build, Makefile to .gitignore and -# making Makefile. -# Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group, record, values=None, ttl=1800): - # print('Here4',file=sys.stderr) + ''' + Wrapper for OctoDNS record. + azuredns.py: + class: octodns.provider.azuredns._AzureRecord + An _AzureRecord is easily accessible to the Azure DNS Management library + functions and is used to wrap all relevant data to create a record in + Azure. + ''' + + def __init__(self, resource_group, record, values=None): + ''' + :param resource_group: The name of resource group in Azure + :type resource_group: str + :param record: An OctoDNS record + :type record: ..record.Record + :param values: Parameters for a record. eg IP address, port, domain + name, etc. Values usually read from record.data + :type values: {'values': [...]} or {'value': [...]} + + :type return: _AzureRecord + ''' self.resource_group = resource_group - self.zone_name = record.zone.name[0:len(record.zone.name)-1] # strips last period + self.zone_name = record.zone.name[0:len(record.zone.name)-1] self.relative_record_set_name = record.name or '@' self.record_type = record._type - type_name = '{}records'.format(self.record_type).lower() + data = values or record.data + format_u_s = '' if record._type == 'A' else '_' + key_name ='{}{}records'.format(self.record_type, format_u_s).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) - if values == None: - return - # TODO: clean up this bit. - data = values or record.data #This should fail if it gets to record.data? It only returns ttl. TODO - - #depending on mult values or not - #TODO: import explicitly. eval() uses for example ARecord from azure.mgmt.dns.models - self.params = {} - try: - self.params = {'ttl':record.ttl or ttl, \ - type_name:[eval(class_name)(ip) for ip in data['values']] or []} - except KeyError: # means that doesn't have multiple values but single value - self.params = {'ttl':record.ttl or ttl, \ - type_name:[eval(class_name)(data['value'])] or []} + self.params = None + if not self.record_type == 'CNAME': + self.params = self._params(data, key_name, eval(class_name)) + else: + self.params = {'cname_record': CnameRecord(data['value'])} + self.params['ttl'] = record.ttl + def _params(self, data, key_name, azure_class): + return {key_name: [azure_class(v) for v in data['values']]} \ + if 'values' in data else {key_name: [azure_class(data['value'])]} - class AzureProvider(BaseProvider): ''' Azure DNS Provider - - azure.py: - class: octodns.provider.azure.AzureProvider + + azuredns.py: + class: octodns.provider.azuredns.AzureProvider # Current support of authentication of access to Azure services only # includes using a Service Principal: # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ @@ -71,90 +83,118 @@ class AzureProvider(BaseProvider): # Resource Group name req: resource_group: - testing: test authentication vars located in /home/t-hehwan/vars.txt + TODO: change the config file to use env variables instead of hard-coded keys? + + personal notes: testing: test authentication vars located in /home/t-hehwan/vars.txt ''' - SUPPORTS_GEO = False # TODO. Will add support as project progresses. + SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) - def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): + def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, + *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, client_id=%s, ' 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) credentials = ServicePrincipalCredentials( - client_id = client_id, secret = key, tenant = directory_id + client_id, secret = key, tenant = directory_id ) self._dns_client = DnsManagementClient(credentials, sub_id) self._resource_group = resource_group - - self._azure_zones = None # will be a dictionary. key: name. val: id. - self._azure_records = {} # will be dict by octodns record, az record - - self._supported_types = ['CNAME', 'A', 'AAAA', 'MX', 'SRV', 'NS', 'PTR'] - # TODO: add TXT - - # TODO: health checks a la route53. - - + self._azure_zones = set() - # TODO: add support for all types. First skeleton: add A. - def supports(self, record): - return record._type in self._supported_types - - @property - def azure_zones(self): - if self._azure_zones is None: - self.log.debug('azure_zones: loading') - zones = {} - for zone in self._dns_client.zones.list_by_resource_group(self._resource_group): - zones[zone.name] = zone.id - self._azure_zones = zones - return self._azure_zones - - # Given a zone name, returns the zone id. If DNE, creates it. - def _get_zone_id(self, name, create=False): - self.log.debug('_get_zone_id: name=%s', name) + def _populate_zones(self): + self.log.debug('azure_zones: loading') + for zone in self._dns_client.zones.list_by_resource_group( + self._resource_group): + self._azure_zones.add(zone.name) + + def _check_zone(self, name, create=False): + ''' + Checks whether a zone specified in a source exist in Azure server. + Note that Azure zones omit end '.' eg: contoso.com vs contoso.com. + Returns the name if it exists. + + :param name: Name of a zone to checks + :type name: str + :param create: If True, creates the zone of that name. + :type create: bool + + :type return: str or None + ''' + self.log.debug('_check_zone: name=%s', name) try: - id = self._dns_client.zones.get(self._resource_group, name) - self.log.debug('_get_zone_id: id=%s', id) - return id + if name in self._azure_zones: + return name + if self._dns_client.zones.get(self._resource_group, name): + self._azure_zones.add(name) + return name except: if create: - self.log.debug('_get_zone_id: no matching zone; creating %s', name) - #TODO: write - return None #placeholder - return None + try: + self.log.debug('_check_zone: no matching zone; creating %s', + name) + if self._dns_client.zones.create_or_update( + self._resource_group, name, Zone('global')): #TODO: figure out what location should be + return name + except: + raise + return None - # Create a dictionary of record objects by zone and octodns record names - # TODO: add geo parsing - def populate(self, zone, target): - zone_name = zone.name[0:len(zone.name)-1]#Azure zone names do not include suffix . + def populate(self, zone, target=False): + ''' + Required function of manager.py. + + Special notes for Azure. Azure zone names omit final '.' + Azure record names for '' are represented by '@' + Azure records created through online interface may have null values + (eg, no IP address for A record). Specific quirks such as these are + responsible for any strange parsing. + + :param zone: A dns zone + :type zone: octodns.zone.Zone + :param target: Checks if Azure is source or target of config. + Currently only supports as a target. Does not use. + :type target: bool + + + TODO: azure interface allows null values. If this attempts to populate with them, will fail. add safety check (simply delete records with null values?) + + :type return: void + ''' + zone_name = zone.name[0:len(zone.name)-1] self.log.debug('populate: name=%s', zone_name) before = len(zone.records) - zone_id = self._get_zone_id(zone_name) - if zone_id: - #records = defaultdict(list) - for type in self._supported_types: - # print('populate. type: {}'.format(type),file=sys.stderr) - for azrecord in self._dns_client.record_sets.list_by_type(self._resource_group, zone_name, type): - # print(azrecord, file=sys.stderr) + + self._populate_zones() + if self._check_zone(zone_name): + for typ in self.SUPPORTS: + for azrecord in self._dns_client.record_sets.list_by_type( + self._resource_group, zone_name, typ): record_name = azrecord.name if azrecord.name != '@' else '' - data = self._type_and_ttl(type, azrecord, - getattr(self, '_data_for_{}'.format(type))(azrecord)) # TODO: azure online interface allows None values. must validate. + data = self._type_and_ttl(typ, azrecord.ttl, + getattr(self, '_data_for_{}'.format(typ))(azrecord)) + record = Record.new(zone, record_name, data, source=self) - # print('HERE0',file=sys.stderr) zone.add_record(record) - self._azure_records[record] = _AzureRecord(self._resource_group, record, data) - # print('HERE1',file=sys.stderr) + self.log.info('populate: found %s records', len(zone.records)-before) - - # might not need - def _get_type(azrecord): - azrecord['type'].split('/')[-1] - - def _type_and_ttl(self, type, azrecord, data): - data['type'] = type - data['ttl'] = azrecord.ttl + + def _type_and_ttl(self, typ, ttl, data): + ''' Adds type and ttl fields to return dictionary. + + :param typ: The type of a record + :type typ: str + :param ttl: The ttl of a record + :type ttl: int + :param data: Dictionary holding values of a record. eg, IP addresses + :type data: {'values': [...]} or {'value': [...]} + + :type return: {...} + ''' + data['type'] = typ + data['ttl'] = ttl return data def _data_for_A(self, azrecord): @@ -164,12 +204,10 @@ class AzureProvider(BaseProvider): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} def _data_for_TXT(self, azrecord): - print('azure',file=sys.stderr) - print([ar.value for ar in azrecord.txt_records], file=sys.stderr) - print('',file=sys.stderr) - return {'values': [ar.value for ar in azrecord.txt_records]} + return {'values': \ + [reduce((lambda a,b:a+b), ar.value) for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): + def _data_for_CNAME(self, azrecord): #TODO: see TODO in population comment. try: val = azrecord.cname_record.cname if not val.endswith('.'): @@ -178,7 +216,7 @@ class AzureProvider(BaseProvider): except: return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. - def _data_for_PTR(self, azrecord): + def _data_for_PTR(self, azrecord): #TODO: see TODO in population comment. try: val = azrecord.ptr_records[0].ptdrname if not val.endswith('.'): @@ -198,21 +236,23 @@ class AzureProvider(BaseProvider): 'target': ar.target} for ar in azrecord.srv_records] } - def _data_for_NS(self, azrecord): + def _data_for_NS(self, azrecord): #TODO: see TODO in population comment. def period_validate(string): return string if string.endswith('.') else string + '.' vals = [ar.nsdname for ar in azrecord.ns_records] return {'values': [period_validate(val) for val in vals]} def _apply_Create(self, change): - new = change.new - - #validate that the zone exists. - #self._get_zone_id(new.name, create=True) - - ar = _AzureRecord(self._resource_group, new, new.data) - + ''' A record from change must be created. + + :param change: a change object + :type change: octodns.record.Change + + :type return: void + ''' + ar = _AzureRecord(self._resource_group, change.new) create = self._dns_client.record_sets.create_or_update + create(resource_group_name=ar.resource_group, zone_name=ar.zone_name, relative_record_set_name=ar.relative_record_set_name, @@ -220,56 +260,33 @@ class AzureProvider(BaseProvider): parameters=ar.params) def _apply_Delete(self, change): - existing = change.existing - ar = _AzureRecord(self._resource_group, existing) + ar = _AzureRecord(self._resource_group, change.existing) delete = self._dns_client.record_sets.delete + delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) def _apply_Update(self, change): - self._apply_Delete(change) self._apply_Create(change) - # type plan: Plan class from .base def _apply(self, plan): + ''' + Required function of manager.py + + :param plan: Contains the zones and changes to be made + :type plan: octodns.provider.base.Plan + + :type return: void + ''' desired = plan.desired changes = plan.changes self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - # validate that the zone exists. function creates zone if DNE. - self._get_zone_id(desired.name) + azure_zone_name = desired.name[0:len(desired.name)-1] + self._check_zone(azure_zone_name, create=True) - # Some parsing bits to call _mod_Create or _mod_Delete. - # changes is a list of Delete and Create objects. for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) - - - # ********** - # Figuring out what object plan is. - - # self._executor = ThreadPoolExecutor(max_workers) - # futures.append(self._executor.submit(self._populate_and_plan, - # zone_name, sources, targets)) - # plans = [p for f in futures for p in f.results()] - # type of plans[0] == type of one output of _populate_and_plan - - # for target, plan in plans: - # apply(plan) - - - # type(target) == BaseProvider - # type(plan) == Plan() - - # Plan(existing, desired, changes) - # existing.type == desired.type == Zone(desired.name, desired.sub_zones) - # Zone(name, sub_zones) (str and set of strs) - # changes.type = [Delete/Create] - - - # Starts with sync in main() of sync. - # {u'values': ['3.3.3.3', '4.4.4.4'], u'type': 'A', u'ttl': 3600} - # {u'type': u'A', u'value': [u'3.3.3.3', u'4.4.4.4'], u'ttl': 3600L} \ No newline at end of file From 5e4d68094fe3e782540e623ab6fd303c37be5c19 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 24 Jun 2017 17:14:48 -0700 Subject: [PATCH 20/84] Add meta record support with provider id to zone Support replace=True in zone.add_record --- octodns/manager.py | 14 +++++++++++++- octodns/zone.py | 7 ++++++- tests/test_octodns_manager.py | 6 ++++++ tests/test_octodns_zone.py | 6 ++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index 07719dd..e6fe253 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -13,6 +13,7 @@ import logging from .provider.base import BaseProvider from .provider.yaml import YamlProvider +from .record import Record from .yaml import safe_load from .zone import Zone @@ -59,7 +60,7 @@ class MainThreadExecutor(object): class Manager(object): log = logging.getLogger('Manager') - def __init__(self, config_file, max_workers=None): + def __init__(self, config_file, max_workers=None, include_meta=False): self.log.info('__init__: config_file=%s', config_file) # Read our config file @@ -75,6 +76,10 @@ class Manager(object): else: self._executor = MainThreadExecutor() + self.include_meta = include_meta or manager_config.get('include_meta', + False) + self.log.info('__init__: max_workers=%s', self.include_meta) + self.log.debug('__init__: configuring providers') self.providers = {} for provider_name, provider_config in self.config['providers'].items(): @@ -176,6 +181,13 @@ class Manager(object): plans = [] for target in targets: + if self.include_meta: + meta = Record.new(zone, 'octodns-meta', { + 'type': 'TXT', + 'ttl': 60, + 'value': 'provider={}'.format(target.id) + }) + zone.add_record(meta, replace=True) plan = target.plan(zone) if plan: plans.append((target, plan)) diff --git a/octodns/zone.py b/octodns/zone.py index 03bc41c..9d405bb 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -49,9 +49,13 @@ class Zone(object): def hostname_from_fqdn(self, fqdn): return self._name_re.sub('', fqdn) - def add_record(self, record): + def add_record(self, record, replace=False): name = record.name last = name.split('.')[-1] + + if replace and record in self.records: + self.records.remove(record) + if last in self.sub_zones: if name != last: # it's a record for something under a sub-zone @@ -63,6 +67,7 @@ class Zone(object): raise SubzoneRecordException('Record {} a managed sub-zone ' 'and not of type NS' .format(record.fqdn)) + # TODO: this is pretty inefficent for existing in self.records: if record == existing: raise DuplicateRecordException('Duplicate record {}, type {}' diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 641c1ff..45a3b55 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -128,6 +128,12 @@ class TestManager(TestCase): .sync(dry_run=False, force=True) self.assertEquals(19, tc) + # Include meta + tc = Manager(get_config_filename('simple.yaml'), max_workers=1, + include_meta=True) \ + .sync(dry_run=False, force=True) + self.assertEquals(23, tc) + def test_eligible_targets(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index a4d7300..f310397 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -39,6 +39,7 @@ class TestZone(TestCase): a = ARecord(zone, 'a', {'ttl': 42, 'value': '1.1.1.1'}) b = ARecord(zone, 'b', {'ttl': 42, 'value': '1.1.1.1'}) + c = ARecord(zone, 'a', {'ttl': 43, 'value': '2.2.2.2'}) zone.add_record(a) self.assertEquals(zone.records, set([a])) @@ -48,6 +49,11 @@ class TestZone(TestCase): self.assertEquals('Duplicate record a.unit.tests., type A', ctx.exception.message) self.assertEquals(zone.records, set([a])) + + # can add duplicate with replace=True + zone.add_record(c, replace=True) + self.assertEquals('2.2.2.2', list(zone.records)[0].values[0]) + # Can add dup name, with different type zone.add_record(b) self.assertEquals(zone.records, set([a, b])) From cc47bd70348ac711da8dc6d3ef2ce4b74a2642b4 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 27 Jun 2017 12:10:57 -0700 Subject: [PATCH 21/84] Fixed bug for MX and SRV. Added Azure test suite as well. --- octodns/provider/azuredns.py | 287 ++++++++++++++---------- rb.txt | 8 - tests/test_octodns_provider_azuredns.py | 171 ++++++++++++++ 3 files changed, 342 insertions(+), 124 deletions(-) delete mode 100755 rb.txt create mode 100644 tests/test_octodns_provider_azuredns.py diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 07b39fe..013613a 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import sys from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient @@ -14,53 +13,120 @@ from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ from functools import reduce import logging -import re -from ..record import Record, Update +from ..record import Record from .base import BaseProvider + class _AzureRecord(object): - ''' - Wrapper for OctoDNS record. - azuredns.py: + ''' Wrapper for OctoDNS record. + azuredns.py: class: octodns.provider.azuredns._AzureRecord - An _AzureRecord is easily accessible to the Azure DNS Management library - functions and is used to wrap all relevant data to create a record in + An _AzureRecord is easily accessible to Azure DNS Management library + functions and is used to wrap all relevant data to create a record in Azure. ''' - + def __init__(self, resource_group, record, values=None): - ''' + ''' :param resource_group: The name of resource group in Azure - :type resource_group: str - :param record: An OctoDNS record + :type resource_group: str + :param record: An OctoDNS record :type record: ..record.Record :param values: Parameters for a record. eg IP address, port, domain name, etc. Values usually read from record.data :type values: {'values': [...]} or {'value': [...]} - + :type return: _AzureRecord ''' self.resource_group = resource_group - self.zone_name = record.zone.name[0:len(record.zone.name)-1] + self.zone_name = record.zone.name[0:len(record.zone.name) - 1] self.relative_record_set_name = record.name or '@' self.record_type = record._type - + data = values or record.data format_u_s = '' if record._type == 'A' else '_' - key_name ='{}{}records'.format(self.record_type, format_u_s).lower() + key_name = '{}{}records'.format(self.record_type, format_u_s).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) - self.params = None - if not self.record_type == 'CNAME': - self.params = self._params(data, key_name, eval(class_name)) - else: - self.params = {'cname_record': CnameRecord(data['value'])} + + self.params = getattr(self, '_params_for_{}'.format(record._type)) + self.params = self.params(data, key_name, eval(class_name)) self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): return {key_name: [azure_class(v) for v in data['values']]} \ if 'values' in data else {key_name: [azure_class(data['value'])]} + _params_for_A = _params + _params_for_AAAA = _params + _params_for_NS = _params + _params_for_PTR = _params + _params_for_TXT = _params + + def _params_for_SRV(self, data, key_name, azure_class): + params = [] + if 'values' in data: + for vals in data['values']: + params.append(azure_class(vals['priority'], + vals['weight'], + vals['port'], + vals['target'])) + else: + params.append(azure_class(data['value']['priority'], + data['value']['weight'], + data['value']['port'], + data['value']['target'])) + return {key_name: params} + + def _params_for_MX(self, data, key_name, azure_class): + params = [] + if 'values' in data: + for vals in data['values']: + params.append(azure_class(vals['priority'], + vals['value'])) + else: + params.append(azure_class(data['value']['priority'], + data['value']['value'])) + return {key_name: params} + + def _params_for_CNAME(self, data, key_name, azure_class): + return {'cname_record': CnameRecord(data['value'])} + + def _equals(self, b): + def parse_dict(params): + vals = [] + for char in params: + if char != 'ttl': + list_records = params[char] + try: + for record in list_records: + vals.append(record.__dict__) + except: + vals.append(list_records.__dict__) + vals.sort() + return vals + + return (self.resource_group == b.resource_group) & \ + (self.zone_name == b.zone_name) & \ + (self.record_type == b.record_type) & \ + (self.params['ttl'] == b.params['ttl']) & \ + (parse_dict(self.params) == parse_dict(b.params)) & \ + (self.relative_record_set_name == b.relative_record_set_name) + + def __str__(self): + string = 'Zone: {}; '.format(self.zone_name) + string += 'Name: {}; '.format(self.relative_record_set_name) + string += 'Type: {}; '.format(self.record_type) + string += 'Ttl: {}; '.format(self.params['ttl']) + for char in self.params: + if char != 'ttl': + try: + for rec in self.params[char]: + string += 'Record: {}; '.format(rec.__dict__) + except: + string += 'Record: {}; '.format(self.params[char].__dict__) + return string + class AzureProvider(BaseProvider): ''' @@ -68,11 +134,11 @@ class AzureProvider(BaseProvider): azuredns.py: class: octodns.provider.azuredns.AzureProvider - # Current support of authentication of access to Azure services only - # includes using a Service Principal: + # Current support of authentication of access to Azure services only + # includes using a Service Principal: # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ # resource-group-create-service-principal-portal - # The Azure Active Directory Application ID (referred to client ID) req: + # The Azure Active Directory Application ID (aka client ID) req: client_id: # Authentication Key Value req: key: @@ -82,32 +148,33 @@ class AzureProvider(BaseProvider): sub_id: # Resource Group name req: resource_group: - - TODO: change the config file to use env variables instead of hard-coded keys? - - personal notes: testing: test authentication vars located in /home/t-hehwan/vars.txt + + TODO: change config file to use env vars instead of hard-coded keys + + personal notes: testing: test authentication vars located in + /home/t-hehwan/vars.txt ''' SUPPORTS_GEO = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) - - def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, - *args, **kwargs): + + def __init__(self, id, client_id, key, directory_id, sub_id, + resource_group, *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, client_id=%s, ' - 'key=***, directory_id:%s', id, client_id, directory_id) + 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) credentials = ServicePrincipalCredentials( - client_id, secret = key, tenant = directory_id + client_id, secret=key, tenant=directory_id ) self._dns_client = DnsManagementClient(credentials, sub_id) self._resource_group = resource_group self._azure_zones = set() - + def _populate_zones(self): self.log.debug('azure_zones: loading') - for zone in self._dns_client.zones.list_by_resource_group( - self._resource_group): + list_zones = self._dns_client.zones.list_by_resource_group + for zone in list_zones(self._resource_group): self._azure_zones.add(zone.name) def _check_zone(self, name, create=False): @@ -115,12 +182,12 @@ class AzureProvider(BaseProvider): Checks whether a zone specified in a source exist in Azure server. Note that Azure zones omit end '.' eg: contoso.com vs contoso.com. Returns the name if it exists. - + :param name: Name of a zone to checks :type name: str :param create: If True, creates the zone of that name. :type create: bool - + :type return: str or None ''' self.log.debug('_check_zone: name=%s', name) @@ -130,163 +197,151 @@ class AzureProvider(BaseProvider): if self._dns_client.zones.get(self._resource_group, name): self._azure_zones.add(name) return name - except: + except: # TODO: figure out what location should be if create: try: - self.log.debug('_check_zone: no matching zone; creating %s', - name) - if self._dns_client.zones.create_or_update( - self._resource_group, name, Zone('global')): #TODO: figure out what location should be + self.log.debug('_check_zone:no matching zone; creating %s', + name) + create_zone = self._dns_client.zones.create_or_update + if create_zone(self._resource_group, name, Zone('global')): return name except: raise return None - - def populate(self, zone, target=False): + + def populate(self, zone, target=False, lenient=False): ''' Required function of manager.py. - + Special notes for Azure. Azure zone names omit final '.' Azure record names for '' are represented by '@' Azure records created through online interface may have null values (eg, no IP address for A record). Specific quirks such as these are responsible for any strange parsing. - + :param zone: A dns zone :type zone: octodns.zone.Zone - :param target: Checks if Azure is source or target of config. + :param target: Checks if Azure is source or target of config. Currently only supports as a target. Does not use. :type target: bool - - - TODO: azure interface allows null values. If this attempts to populate with them, will fail. add safety check (simply delete records with null values?) - + + + TODO: azure interface allows null values. If this attempts to + populate with them, will fail. add safety check (simply delete + records with null values?) + :type return: void ''' - zone_name = zone.name[0:len(zone.name)-1] + zone_name = zone.name[0:len(zone.name) - 1] self.log.debug('populate: name=%s', zone_name) before = len(zone.records) self._populate_zones() if self._check_zone(zone_name): for typ in self.SUPPORTS: - for azrecord in self._dns_client.record_sets.list_by_type( - self._resource_group, zone_name, typ): + records = self._dns_client.record_sets.list_by_type + for azrecord in records(self._resource_group, zone_name, typ): record_name = azrecord.name if azrecord.name != '@' else '' - data = self._type_and_ttl(typ, azrecord.ttl, - getattr(self, '_data_for_{}'.format(typ))(azrecord)) - + data = getattr(self, '_data_for_{}'.format(typ))(azrecord) + data['type'] = typ + data['ttl'] = azrecord.ttl + record = Record.new(zone, record_name, data, source=self) zone.add_record(record) - self.log.info('populate: found %s records', len(zone.records)-before) + self.log.info('populate: found %s records', len(zone.records) - before) - def _type_and_ttl(self, typ, ttl, data): - ''' Adds type and ttl fields to return dictionary. - - :param typ: The type of a record - :type typ: str - :param ttl: The ttl of a record - :type ttl: int - :param data: Dictionary holding values of a record. eg, IP addresses - :type data: {'values': [...]} or {'value': [...]} - - :type return: {...} - ''' - data['type'] = typ - data['ttl'] = ttl - return data - def _data_for_A(self, azrecord): return {'values': [ar.ipv4_address for ar in azrecord.arecords]} - + def _data_for_AAAA(self, azrecord): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} - - def _data_for_TXT(self, azrecord): - return {'values': \ - [reduce((lambda a,b:a+b), ar.value) for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): #TODO: see TODO in population comment. + def _data_for_TXT(self, azrecord): + return {'values': [reduce((lambda a, b: a + b), ar.value) + for ar in azrecord.txt_records]} + + def _data_for_CNAME(self, azrecord): # TODO: see TODO in pop comment. try: val = azrecord.cname_record.cname if not val.endswith('.'): val += '.' return {'value': val} except: - return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. - - def _data_for_PTR(self, azrecord): #TODO: see TODO in population comment. + return {'value': '.'} # TODO: this is a bad fix. but octo checks + # that cnames have trailing '.' while azure allows creating cnames + # on the online interface with no value. + + def _data_for_PTR(self, azrecord): # TODO: see TODO in population comment. try: val = azrecord.ptr_records[0].ptdrname if not val.endswith('.'): val += '.' return {'value': val} except: - return {'value': '.' } #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. - + return {'value': '.'} + def _data_for_MX(self, azrecord): - return {'values': [{'priority':ar.preference, - 'value':ar.exchange} for ar in azrecord.mx_records]} - + return {'values': [{'priority': ar.preference, 'value': ar.exchange} + for ar in azrecord.mx_records] + } + def _data_for_SRV(self, azrecord): - return {'values': [{'priority': ar.priority, - 'weight': ar.weight, - 'port': ar.port, - 'target': ar.target} for ar in azrecord.srv_records] - } - - def _data_for_NS(self, azrecord): #TODO: see TODO in population comment. + return {'values': [{'priority': ar.priority, 'weight': ar.weight, + 'port': ar.port, 'target': ar.target} + for ar in azrecord.srv_records] + } + + def _data_for_NS(self, azrecord): # TODO: see TODO in population comment. def period_validate(string): return string if string.endswith('.') else string + '.' vals = [ar.nsdname for ar in azrecord.ns_records] return {'values': [period_validate(val) for val in vals]} def _apply_Create(self, change): - ''' A record from change must be created. - + '''A record from change must be created. + :param change: a change object :type change: octodns.record.Change - + :type return: void ''' ar = _AzureRecord(self._resource_group, change.new) create = self._dns_client.record_sets.create_or_update - - create(resource_group_name=ar.resource_group, - zone_name=ar.zone_name, - relative_record_set_name=ar.relative_record_set_name, - record_type=ar.record_type, + + create(resource_group_name=ar.resource_group, + zone_name=ar.zone_name, + relative_record_set_name=ar.relative_record_set_name, + record_type=ar.record_type, parameters=ar.params) - + def _apply_Delete(self, change): ar = _AzureRecord(self._resource_group, change.existing) delete = self._dns_client.record_sets.delete - - delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, + + delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - + def _apply_Update(self, change): self._apply_Create(change) - + def _apply(self, plan): ''' Required function of manager.py - + :param plan: Contains the zones and changes to be made :type plan: octodns.provider.base.Plan - + :type return: void ''' desired = plan.desired changes = plan.changes - self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - - azure_zone_name = desired.name[0:len(desired.name)-1] + + azure_zone_name = desired.name[0:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) - + for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) - diff --git a/rb.txt b/rb.txt deleted file mode 100755 index d6a7949..0000000 --- a/rb.txt +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -#script to rebuild octodns quickly - -sudo rm -r /home/t-hehwan/GitHub/octodns/build -sudo rm -r /home/t-hehwan/GitHub/octodns/octodns.egg-info -sudo python /home/t-hehwan/GitHub/octodns/setup.py -q build -sudo python /home/t-hehwan/GitHub/octodns/setup.py -q install -octodns-sync --config-file=./config/production.yaml \ No newline at end of file diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py new file mode 100644 index 0000000..db4d293 --- /dev/null +++ b/tests/test_octodns_provider_azuredns.py @@ -0,0 +1,171 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from octodns.record import Create, Delete, Record, Update +from octodns.provider.azuredns import _AzureRecord, AzureProvider +from octodns.zone import Zone + +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone as AzureZone + +from octodns.zone import Zone + +from unittest import TestCase +import sys + + +class Test_AzureRecord(TestCase): + zone = Zone(name='unit.tests.', sub_zones=[]) + octo_records = [] + octo_records.append(Record.new(zone, '', { + 'ttl': 0, + 'type': 'A', + 'values': ['1.2.3.4', '10.10.10.10'] + })) + octo_records.append(Record.new(zone, 'a', { + 'ttl': 1, + 'type': 'A', + 'values': ['1.2.3.4', '1.1.1.1'], + })) + octo_records.append(Record.new(zone, 'aa', { + 'ttl': 9001, + 'type': 'A', + 'values': ['1.2.4.3'] + })) + octo_records.append(Record.new(zone, 'aaa', { + 'ttl': 2, + 'type': 'A', + 'values': ['1.1.1.3'] + })) + octo_records.append(Record.new(zone, 'cname', { + 'ttl': 3, + 'type': 'CNAME', + 'value': 'a.unit.tests.', + })) + octo_records.append(Record.new(zone, '', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }, { + 'priority': 20, + 'value': 'mx2.unit.tests.', + }] + })) + octo_records.append(Record.new(zone, '', { + 'ttl': 4, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.'], + })) + octo_records.append(Record.new(zone, '', { + 'ttl': 5, + 'type': 'NS', + 'value': 'ns1.unit.tests.', + })) + octo_records.append(Record.new(zone, '_srv._tcp', { + 'ttl': 6, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 12, + 'weight': 30, + 'port': 30, + 'target': 'foo-2.unit.tests.', + }] + })) + + azure_records = [] + _base0 = _AzureRecord('TestAzure', octo_records[0]) + _base0.zone_name = 'unit.tests' + _base0.relative_record_set_name = '@' + _base0.record_type = 'A' + _base0.params['ttl'] = 0 + _base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')] + azure_records.append(_base0) + + _base1 = _AzureRecord('TestAzure', octo_records[1]) + _base1.zone_name = 'unit.tests' + _base1.relative_record_set_name = 'a' + _base1.record_type = 'A' + _base1.params['ttl'] = 1 + _base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')] + azure_records.append(_base1) + + _base2 = _AzureRecord('TestAzure', octo_records[2]) + _base2.zone_name = 'unit.tests' + _base2.relative_record_set_name = 'aa' + _base2.record_type = 'A' + _base2.params['ttl'] = 9001 + _base2.params['arecords'] = ARecord('1.2.4.3') + azure_records.append(_base2) + + _base3 = _AzureRecord('TestAzure', octo_records[3]) + _base3.zone_name = 'unit.tests' + _base3.relative_record_set_name = 'aaa' + _base3.record_type = 'A' + _base3.params['ttl'] = 2 + _base3.params['arecords'] = ARecord('1.1.1.3') + azure_records.append(_base3) + + _base4 = _AzureRecord('TestAzure', octo_records[4]) + _base4.zone_name = 'unit.tests' + _base4.relative_record_set_name = 'cname' + _base4.record_type = 'CNAME' + _base4.params['ttl'] = 3 + _base4.params['cname_record'] = CnameRecord('a.unit.tests.') + azure_records.append(_base4) + + _base5 = _AzureRecord('TestAzure', octo_records[5]) + _base5.zone_name = 'unit.tests' + _base5.relative_record_set_name = '@' + _base5.record_type = 'MX' + _base5.params['ttl'] = 3 + _base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'), + MxRecord(20, 'mx2.unit.tests.')] + azure_records.append(_base5) + + _base6 = _AzureRecord('TestAzure', octo_records[6]) + _base6.zone_name = 'unit.tests' + _base6.relative_record_set_name = '@' + _base6.record_type = 'NS' + _base6.params['ttl'] = 4 + _base6.params['ns_records'] = [NsRecord('ns1.unit.tests.'), + NsRecord('ns2.unit.tests.')] + azure_records.append(_base6) + + _base7 = _AzureRecord('TestAzure', octo_records[7]) + _base7.zone_name = 'unit.tests' + _base7.relative_record_set_name = '@' + _base7.record_type = 'NS' + _base7.params['ttl'] = 5 + _base7.params['ns_records'] = [NsRecord('ns1.unit.tests.')] + azure_records.append(_base7) + + _base8 = _AzureRecord('TestAzure', octo_records[8]) + _base8.zone_name = 'unit.tests' + _base8.relative_record_set_name = '_srv._tcp' + _base8.record_type = 'SRV' + _base8.params['ttl'] = 6 + _base8.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'), + SrvRecord(12, 30, 30, 'foo-2.unit.tests.')] + azure_records.append(_base8) + + def test_azure_record(self): + assert(len(self.azure_records) == len(self.octo_records)) + for i in range(len(self.azure_records)): + octo = _AzureRecord('TestAzure', self.octo_records[i]) + assert(self.azure_records[i]._equals(octo)) + + +class TestAzureDnsProvider(TestCase): + def test_populate(self): + pass # placeholder From 3a2ccdcac09497540ce8d681a3ae33c96fd3f389 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 28 Jun 2017 03:09:41 -0700 Subject: [PATCH 22/84] Manually join self.values to avoid double escapes, e.g. \\; --- octodns/record.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/octodns/record.py b/octodns/record.py index 23bfd45..6ee9dff 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -254,9 +254,10 @@ class _ValuesMixin(object): return ret def __repr__(self): + values = "['{}']".format("', '".join([str(v) for v in self.values])) return '<{} {} {}, {}, {}>'.format(self.__class__.__name__, self._type, self.ttl, - self.fqdn, self.values) + self.fqdn, values) class _GeoMixin(_ValuesMixin): From 0fb88a959a156ef2a4be6cf6ef00b70ccabfe3b6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 28 Jun 2017 03:26:23 -0700 Subject: [PATCH 23/84] Add retry to ns1 provider --- octodns/provider/ns1.py | 29 +++++++++++++++++++++++++---- requirements.txt | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 2f0a024..3b8ad00 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -7,7 +7,8 @@ from __future__ import absolute_import, division, print_function, \ from logging import getLogger from nsone import NSONE -from nsone.rest.errors import ResourceException +from nsone.rest.errors import RateLimitException, ResourceException +from time import sleep from ..record import Record from .base import BaseProvider @@ -25,6 +26,7 @@ class Ns1Provider(BaseProvider): SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) + RATE_LIMIT_DELAY = 1 ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' def __init__(self, id, api_key, *args, **kwargs): @@ -171,7 +173,14 @@ class Ns1Provider(BaseProvider): name = self._get_name(new) _type = new._type params = getattr(self, '_params_for_{}'.format(_type))(new) - getattr(nsone_zone, 'add_{}'.format(_type))(name, **params) + meth = getattr(nsone_zone, 'add_{}'.format(_type)) + try: + meth(name, **params) + except RateLimitException: + self.log.warn('_apply_Create: rate limit encountered, pausing ' + 'and trying again') + sleep(self.RATE_LIMIT_DELAY) + meth(name, **params) def _apply_Update(self, nsone_zone, change): existing = change.existing @@ -180,14 +189,26 @@ class Ns1Provider(BaseProvider): record = nsone_zone.loadRecord(name, _type) new = change.new params = getattr(self, '_params_for_{}'.format(_type))(new) - record.update(**params) + try: + record.update(**params) + except RateLimitException: + self.log.warn('_apply_Update: rate limit encountered, pausing ' + 'and trying again') + sleep(self.RATE_LIMIT_DELAY) + record.update(**params) def _apply_Delete(self, nsone_zone, change): existing = change.existing name = self._get_name(existing) _type = existing._type record = nsone_zone.loadRecord(name, _type) - record.delete() + try: + record.delete() + except RateLimitException: + self.log.warn('_apply_Delete: rate limit encountered, pausing ' + 'and trying again') + sleep(self.RATE_LIMIT_DELAY) + record.delete() def _apply(self, plan): desired = plan.desired diff --git a/requirements.txt b/requirements.txt index b10ca4c..93a8521 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ incf.countryutils==1.0 ipaddress==1.0.18 jmespath==0.9.0 natsort==5.0.3 -nsone==0.9.10 +nsone==0.9.14 python-dateutil==2.6.0 requests==2.13.0 s3transfer==0.1.10 From a44b82c2c79e6025c605f77fd797c2254c2939dd Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 28 Jun 2017 04:11:46 -0700 Subject: [PATCH 24/84] NS1 rate_limit_delay param, unit tests for rate limit handling --- octodns/provider/ns1.py | 13 ++++++----- tests/test_octodns_provider_ns1.py | 35 +++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 3b8ad00..0f3db1f 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -26,14 +26,15 @@ class Ns1Provider(BaseProvider): SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) - RATE_LIMIT_DELAY = 1 ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' - def __init__(self, id, api_key, *args, **kwargs): + def __init__(self, id, api_key, rate_limit_delay=1, *args, **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) - self.log.debug('__init__: id=%s, api_key=***', id) + self.log.debug('__init__: id=%s, api_key=***, rate_limit_delay=%d', id, + rate_limit_delay) super(Ns1Provider, self).__init__(id, *args, **kwargs) self._client = NSONE(apiKey=api_key) + self.rate_limit_delay = rate_limit_delay def _data_for_A(self, _type, record): return { @@ -179,7 +180,7 @@ class Ns1Provider(BaseProvider): except RateLimitException: self.log.warn('_apply_Create: rate limit encountered, pausing ' 'and trying again') - sleep(self.RATE_LIMIT_DELAY) + sleep(self.rate_limit_delay) meth(name, **params) def _apply_Update(self, nsone_zone, change): @@ -194,7 +195,7 @@ class Ns1Provider(BaseProvider): except RateLimitException: self.log.warn('_apply_Update: rate limit encountered, pausing ' 'and trying again') - sleep(self.RATE_LIMIT_DELAY) + sleep(self.rate_limit_delay) record.update(**params) def _apply_Delete(self, nsone_zone, change): @@ -207,7 +208,7 @@ class Ns1Provider(BaseProvider): except RateLimitException: self.log.warn('_apply_Delete: rate limit encountered, pausing ' 'and trying again') - sleep(self.RATE_LIMIT_DELAY) + sleep(self.rate_limit_delay) record.delete() def _apply(self, plan): diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index ecc107c..5e53cfd 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -6,7 +6,8 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from mock import Mock, call, patch -from nsone.rest.errors import AuthException, ResourceException +from nsone.rest.errors import AuthException, RateLimitException, \ + ResourceException from unittest import TestCase from octodns.record import Delete, Record, Update @@ -192,7 +193,7 @@ class TestNs1Provider(TestCase): @patch('nsone.NSONE.createZone') @patch('nsone.NSONE.loadZone') def test_sync(self, load_mock, create_mock): - provider = Ns1Provider('test', 'api-key') + provider = Ns1Provider('test', 'api-key', rate_limit_delay=0) desired = Zone('unit.tests.', []) desired.records.update(self.expected) @@ -225,7 +226,15 @@ class TestNs1Provider(TestCase): create_mock.reset_mock() load_mock.side_effect = \ ResourceException('server error: zone not found') - create_mock.side_effect = None + # ugh, need a mock zone with a mock prop since we're using getattr, we + # can actually control side effects on `meth` with that. + mock_zone = Mock() + mock_zone.add_SRV = Mock() + mock_zone.add_SRV.side_effect = [ + RateLimitException('boo'), + None, + ] + create_mock.side_effect = [mock_zone] got_n = provider.apply(plan) self.assertEquals(expected_n, got_n) @@ -245,12 +254,26 @@ class TestNs1Provider(TestCase): self.assertEquals(2, len(plan.changes)) self.assertIsInstance(plan.changes[0], Update) self.assertIsInstance(plan.changes[1], Delete) - + # ugh, we need a mock record that can be returned from loadRecord for + # the update and delete targets, we can add our side effects to that to + # trigger rate limit handling + mock_record = Mock() + mock_record.update.side_effect = [ + RateLimitException('one'), + None, + ] + mock_record.delete.side_effect = [ + RateLimitException('two'), + None, + ] + nsone_zone.loadRecord.side_effect = [mock_record, mock_record] got_n = provider.apply(plan) self.assertEquals(2, got_n) nsone_zone.loadRecord.assert_has_calls([ call('unit.tests', u'A'), - call().update(answers=[u'1.2.3.4'], ttl=32), call('delete-me', u'A'), - call().delete() + ]) + mock_record.assert_has_calls([ + call.update(answers=[u'1.2.3.4'], ttl=32), + call.delete() ]) From 67c2f9767ba7e74c66f21fbe938050ef86a7007c Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 28 Jun 2017 04:46:59 -0700 Subject: [PATCH 25/84] CHANGELOG, version bump, pass at release script --- CHANGELOG.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ octodns/__init__.py | 2 +- script/release | 12 ++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md create mode 100755 script/release diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1cb1204 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,44 @@ + +## v0.8.4 - 2017-03-14 - It's been too long + +Lots of updates based on our internal use, needs, and feedback & suggestions +from our OSS users. There's too much to list out since the previous release was +cut, but I'll try to cover the highlights/important bits and promise to do +better in the future :fingers_crossed: + +#### Major: + +* Complete rework of record validation with lenient mode support added to + octodns-dump so that data with validation problems can be dumped to config + files as a starting point. octoDNS now also ignores validation errors when + pulling the current state from a provider before planning changes. In both + cases this is best effort. +* Naming of record keys are based on RFC-1035 and friends, previous names have + been kept for backwards compatibility until the 1.0 release. +* Provider record type support is now explicit, i.e. opt-in, rather than + opt-out. This prevents bugs/oversights in record handling where providers + don't support (new) record types and didn't correctly ignore them. +* ALIAS support for DNSimple, Dyn, NS1, PowerDNS +* Ignored record support added, `octodns:\n ignored: True` +* Ns1Provider added + +#### Miscellaneous + +* Use a 3rd party lib for nautrual sorting of keys, rather than my old + implementation. Sorting can be disabled in the YamlProvider with + `enforce_order: False`. +* Semi-colon/escaping fixes and improvements. +* Meta record support, `TXT octodns-meta.`. For now just + `provider=`. Optionally turned on with `include_meta` manager + config val. +* Validations check for CNAMEs co-existing with other records and error out if + found. Was a common mistaken/unknown issue and this surfaces the problem + early. +* Sizeable refactor in the way Route53 record translation works to make it + cleaner/less hacky +* Lots of docs type-o fixes +* Fixed some pretty major bugs in DnsimpleProvider +* Relax UnsafePlan checks a bit, more to come here +* Set User-Agent header on Dyn health checks + +## v0.8.0 - 2017-03-14 - First public release diff --git a/octodns/__init__.py b/octodns/__init__.py index 4806766..b6287e5 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -5,4 +5,4 @@ OctoDNS: DNS as code - Tools for managing DNS across multiple providers from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.8.0' +__VERSION__ = '0.8.4' diff --git a/script/release b/script/release new file mode 100755 index 0000000..16e7641 --- /dev/null +++ b/script/release @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +cd "$(dirname $0)"/.. +ROOT=$(pwd) + +VERSION=$(grep __VERSION__ $ROOT/octodns/__init__.py | sed -e "s/.* = '//" -e "s/'$//") + +git tag -s v$VERSION -m "Release $VERSION" +python setup.py sdist upload +echo "Updloaded $VERSION" From 08d3fda99ef17e4ae0ca529989c14a356b8f925c Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Wed, 28 Jun 2017 14:00:59 -0700 Subject: [PATCH 26/84] safety check for azure null values --- octodns/provider/azuredns.py | 108 ++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 45 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 013613a..6a2db41 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -11,7 +11,7 @@ from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone from functools import reduce - +import sys import logging from ..record import Record from .base import BaseProvider @@ -26,7 +26,7 @@ class _AzureRecord(object): Azure. ''' - def __init__(self, resource_group, record, values=None): + def __init__(self, resource_group, record, delete=False): ''' :param resource_group: The name of resource group in Azure :type resource_group: str @@ -43,14 +43,16 @@ class _AzureRecord(object): self.relative_record_set_name = record.name or '@' self.record_type = record._type - data = values or record.data + if delete: + return + format_u_s = '' if record._type == 'A' else '_' key_name = '{}{}records'.format(self.record_type, format_u_s).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) self.params = getattr(self, '_params_for_{}'.format(record._type)) - self.params = self.params(data, key_name, eval(class_name)) + self.params = self.params(record.data, key_name, eval(class_name)) self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): @@ -71,7 +73,7 @@ class _AzureRecord(object): vals['weight'], vals['port'], vals['target'])) - else: + else: # single value at key 'value' params.append(azure_class(data['value']['priority'], data['value']['weight'], data['value']['port'], @@ -82,17 +84,23 @@ class _AzureRecord(object): params = [] if 'values' in data: for vals in data['values']: - params.append(azure_class(vals['priority'], - vals['value'])) - else: - params.append(azure_class(data['value']['priority'], - data['value']['value'])) + params.append(azure_class(vals['preference'], + vals['exchange'])) + else: # single value at key 'value' + params.append(azure_class(data['value']['preference'], + data['value']['exchange'])) return {key_name: params} def _params_for_CNAME(self, data, key_name, azure_class): return {'cname_record': CnameRecord(data['value'])} def _equals(self, b): + '''Checks whether two records are equal by comparing all fields. + :param b: Another _AzureRecord object + :type b: _AzureRecord + + :type return: bool + ''' def parse_dict(params): vals = [] for char in params: @@ -114,6 +122,9 @@ class _AzureRecord(object): (self.relative_record_set_name == b.relative_record_set_name) def __str__(self): + '''String representation of an _AzureRecord. + :type return: str + ''' string = 'Zone: {}; '.format(self.zone_name) string += 'Name: {}; '.format(self.relative_record_set_name) string += 'Type: {}; '.format(self.record_type) @@ -128,6 +139,10 @@ class _AzureRecord(object): return string +def period_validate(string): + return string if string.endswith('.') else string + '.' + + class AzureProvider(BaseProvider): ''' Azure DNS Provider @@ -213,22 +228,24 @@ class AzureProvider(BaseProvider): ''' Required function of manager.py. - Special notes for Azure. Azure zone names omit final '.' - Azure record names for '' are represented by '@' + Special notes for Azure. + Azure zone names omit final '.' + Azure root records names are represented by '@'. OctoDNS uses '' Azure records created through online interface may have null values - (eg, no IP address for A record). Specific quirks such as these are - responsible for any strange parsing. + (eg, no IP address for A record). + Azure online interface allows constructing records with null values + which are destroyed by _apply. + + Specific quirks such as these are responsible for any non-obvious + parsing in this function and the functions '_params_for_*'. :param zone: A dns zone :type zone: octodns.zone.Zone :param target: Checks if Azure is source or target of config. Currently only supports as a target. Does not use. :type target: bool - - - TODO: azure interface allows null values. If this attempts to - populate with them, will fail. add safety check (simply delete - records with null values?) + :param lenient: Unused. Check octodns.manager for usage. + :type lenient: bool :type return: void ''' @@ -261,40 +278,38 @@ class AzureProvider(BaseProvider): return {'values': [reduce((lambda a, b: a + b), ar.value) for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): # TODO: see TODO in pop comment. - try: - val = azrecord.cname_record.cname - if not val.endswith('.'): - val += '.' - return {'value': val} - except: - return {'value': '.'} # TODO: this is a bad fix. but octo checks - # that cnames have trailing '.' while azure allows creating cnames - # on the online interface with no value. + def _data_for_CNAME(self, azrecord): + '''Parsing data from Azure DNS Client record call + :param azrecord: a return of a call to list azure records + :type azrecord: azure.mgmt.dns.models.RecordSet - def _data_for_PTR(self, azrecord): # TODO: see TODO in population comment. + :type return: dict + + CNAME and PTR both use the catch block to catch possible empty + records. Refer to population comment. + ''' try: - val = azrecord.ptr_records[0].ptdrname - if not val.endswith('.'): - val += '.' - return {'value': val} + return {'value': period_validate(azrecord.cname_record.cname)} + except: + return {'value': '.'} + + def _data_for_PTR(self, azrecord): + try: + return {'value': period_validate(azrecord.ptr_records[0].ptdrname)} except: return {'value': '.'} def _data_for_MX(self, azrecord): - return {'values': [{'priority': ar.preference, 'value': ar.exchange} - for ar in azrecord.mx_records] - } + return {'values': [{'preference': ar.preference, + 'exchange': ar.exchange} + for ar in azrecord.mx_records]} def _data_for_SRV(self, azrecord): return {'values': [{'priority': ar.priority, 'weight': ar.weight, 'port': ar.port, 'target': ar.target} - for ar in azrecord.srv_records] - } + for ar in azrecord.srv_records]} - def _data_for_NS(self, azrecord): # TODO: see TODO in population comment. - def period_validate(string): - return string if string.endswith('.') else string + '.' + def _data_for_NS(self, azrecord): vals = [ar.nsdname for ar in azrecord.ns_records] return {'values': [period_validate(val) for val in vals]} @@ -315,15 +330,18 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) + print('* Success Create/Update: {}'.format(ar), file=sys.stderr) + + _apply_Update = _apply_Create + def _apply_Delete(self, change): - ar = _AzureRecord(self._resource_group, change.existing) + ar = _AzureRecord(self._resource_group, change.existing, delete=True) delete = self._dns_client.record_sets.delete delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - def _apply_Update(self, change): - self._apply_Create(change) + print('* Success Delete: {}'.format(ar), file=sys.stderr) def _apply(self, plan): ''' From b0de5de445f5cffdba2760939c9d9eb96c97b2bb Mon Sep 17 00:00:00 2001 From: anthonyvia Date: Thu, 29 Jun 2017 09:55:52 -0700 Subject: [PATCH 27/84] Supply 'Marker' to Route53 client when paging in order to correctly retrieve the next page of results. --- octodns/provider/route53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index b4fcab8..490d630 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -250,7 +250,7 @@ class Route53Provider(BaseProvider): more = True start = {} while more: - resp = self._conn.list_hosted_zones() + resp = self._conn.list_hosted_zones(**start) for z in resp['HostedZones']: zones[z['Name']] = z['Id'] more = resp['IsTruncated'] From 0b2275c4e6388d25a28e501152fae4a88f2cf0a1 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 30 Jun 2017 11:06:42 -0700 Subject: [PATCH 28/84] Added complete error testing suite for azuredns --- octodns/provider/azuredns.py | 62 ++-- tests/test_octodns_provider_azuredns.py | 456 ++++++++++++++++-------- 2 files changed, 338 insertions(+), 180 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 6a2db41..cf0e2d5 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -128,6 +128,8 @@ class _AzureRecord(object): string = 'Zone: {}; '.format(self.zone_name) string += 'Name: {}; '.format(self.relative_record_set_name) string += 'Type: {}; '.format(self.record_type) + if not hasattr(self, 'params'): + return string string += 'Ttl: {}; '.format(self.params['ttl']) for char in self.params: if char != 'ttl': @@ -139,7 +141,7 @@ class _AzureRecord(object): return string -def period_validate(string): +def _validate_per(string): return string if string.endswith('.') else string + '.' @@ -209,20 +211,18 @@ class AzureProvider(BaseProvider): try: if name in self._azure_zones: return name - if self._dns_client.zones.get(self._resource_group, name): - self._azure_zones.add(name) - return name - except: # TODO: figure out what location should be + self._dns_client.zones.get(self._resource_group, name) + self._azure_zones.add(name) + return name + except: if create: - try: - self.log.debug('_check_zone:no matching zone; creating %s', - name) - create_zone = self._dns_client.zones.create_or_update - if create_zone(self._resource_group, name, Zone('global')): - return name - except: - raise - return None + self.log.debug('_check_zone:no matching zone; creating %s', + name) + create_zone = self._dns_client.zones.create_or_update + create_zone(self._resource_group, name, Zone('global')) + return name + else: + raise def populate(self, zone, target=False, lenient=False): ''' @@ -254,17 +254,21 @@ class AzureProvider(BaseProvider): before = len(zone.records) self._populate_zones() - if self._check_zone(zone_name): - for typ in self.SUPPORTS: - records = self._dns_client.record_sets.list_by_type - for azrecord in records(self._resource_group, zone_name, typ): - record_name = azrecord.name if azrecord.name != '@' else '' - data = getattr(self, '_data_for_{}'.format(typ))(azrecord) - data['type'] = typ - data['ttl'] = azrecord.ttl + self._check_zone(zone_name) - record = Record.new(zone, record_name, data, source=self) - zone.add_record(record) + _records = set() + records = self._dns_client.record_sets.list_by_dns_zone + for azrecord in records(self._resource_group, zone_name): + if azrecord.type in self.SUPPORTS: + _records.add(azrecord) + for azrecord in _records: + record_name = azrecord.name if azrecord.name != '@' else '' + data = getattr(self, '_data_for_{}'.format(azrecord.type)) + data = data(azrecord) + data['type'] = azrecord.type + data['ttl'] = azrecord.ttl + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) self.log.info('populate: found %s records', len(zone.records) - before) @@ -289,13 +293,13 @@ class AzureProvider(BaseProvider): records. Refer to population comment. ''' try: - return {'value': period_validate(azrecord.cname_record.cname)} + return {'value': _validate_per(azrecord.cname_record.cname)} except: return {'value': '.'} def _data_for_PTR(self, azrecord): try: - return {'value': period_validate(azrecord.ptr_records[0].ptdrname)} + return {'value': _validate_per(azrecord.ptr_records[0].ptdrname)} except: return {'value': '.'} @@ -311,7 +315,7 @@ class AzureProvider(BaseProvider): def _data_for_NS(self, azrecord): vals = [ar.nsdname for ar in azrecord.ns_records] - return {'values': [period_validate(val) for val in vals]} + return {'values': [_validate_per(val) for val in vals]} def _apply_Create(self, change): '''A record from change must be created. @@ -330,7 +334,7 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - print('* Success Create/Update: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Create/Update: {}'.format(ar)) _apply_Update = _apply_Create @@ -341,7 +345,7 @@ class AzureProvider(BaseProvider): delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - print('* Success Delete: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): ''' diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index db4d293..edb2db2 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -5,167 +5,321 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from octodns.record import Create, Delete, Record, Update -from octodns.provider.azuredns import _AzureRecord, AzureProvider +from octodns.record import Create, Delete, Record +from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ + _validate_per from octodns.zone import Zone +from octodns.provider.base import Plan from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ - SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone as AzureZone + SrvRecord, NsRecord, PtrRecord, TxtRecord, RecordSet, SoaRecord, \ + Zone as AzureZone +from msrestazure.azure_exceptions import CloudError -from octodns.zone import Zone from unittest import TestCase -import sys +from mock import Mock, patch + + +zone = Zone(name='unit.tests.', sub_zones=[]) +octo_records = [] +octo_records.append(Record.new(zone, '', { + 'ttl': 0, + 'type': 'A', + 'values': ['1.2.3.4', '10.10.10.10']})) +octo_records.append(Record.new(zone, 'a', { + 'ttl': 1, + 'type': 'A', + 'values': ['1.2.3.4', '1.1.1.1']})) +octo_records.append(Record.new(zone, 'aa', { + 'ttl': 9001, + 'type': 'A', + 'values': ['1.2.4.3']})) +octo_records.append(Record.new(zone, 'aaa', { + 'ttl': 2, + 'type': 'A', + 'values': ['1.1.1.3']})) +octo_records.append(Record.new(zone, 'cname', { + 'ttl': 3, + 'type': 'CNAME', + 'value': 'a.unit.tests.'})) +octo_records.append(Record.new(zone, 'mx1', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }, { + 'priority': 20, + 'value': 'mx2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, 'mx2', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '', { + 'ttl': 4, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.']})) +octo_records.append(Record.new(zone, 'foo', { + 'ttl': 5, + 'type': 'NS', + 'value': 'ns1.unit.tests.'})) +octo_records.append(Record.new(zone, '_srv._tcp', { + 'ttl': 6, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 12, + 'weight': 30, + 'port': 30, + 'target': 'foo-2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '_srv2._tcp', { + 'ttl': 7, + 'type': 'SRV', + 'values': [{ + 'priority': 12, + 'weight': 17, + 'port': 1, + 'target': 'srvfoo.unit.tests.', + }]})) + +azure_records = [] +_base0 = _AzureRecord('TestAzure', octo_records[0]) +_base0.zone_name = 'unit.tests' +_base0.relative_record_set_name = '@' +_base0.record_type = 'A' +_base0.params['ttl'] = 0 +_base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')] +azure_records.append(_base0) + +_base1 = _AzureRecord('TestAzure', octo_records[1]) +_base1.zone_name = 'unit.tests' +_base1.relative_record_set_name = 'a' +_base1.record_type = 'A' +_base1.params['ttl'] = 1 +_base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')] +azure_records.append(_base1) + +_base2 = _AzureRecord('TestAzure', octo_records[2]) +_base2.zone_name = 'unit.tests' +_base2.relative_record_set_name = 'aa' +_base2.record_type = 'A' +_base2.params['ttl'] = 9001 +_base2.params['arecords'] = ARecord('1.2.4.3') +azure_records.append(_base2) + +_base3 = _AzureRecord('TestAzure', octo_records[3]) +_base3.zone_name = 'unit.tests' +_base3.relative_record_set_name = 'aaa' +_base3.record_type = 'A' +_base3.params['ttl'] = 2 +_base3.params['arecords'] = ARecord('1.1.1.3') +azure_records.append(_base3) + +_base4 = _AzureRecord('TestAzure', octo_records[4]) +_base4.zone_name = 'unit.tests' +_base4.relative_record_set_name = 'cname' +_base4.record_type = 'CNAME' +_base4.params['ttl'] = 3 +_base4.params['cname_record'] = CnameRecord('a.unit.tests.') +azure_records.append(_base4) + +_base5 = _AzureRecord('TestAzure', octo_records[5]) +_base5.zone_name = 'unit.tests' +_base5.relative_record_set_name = 'mx1' +_base5.record_type = 'MX' +_base5.params['ttl'] = 3 +_base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'), + MxRecord(20, 'mx2.unit.tests.')] +azure_records.append(_base5) + +_base6 = _AzureRecord('TestAzure', octo_records[6]) +_base6.zone_name = 'unit.tests' +_base6.relative_record_set_name = 'mx2' +_base6.record_type = 'MX' +_base6.params['ttl'] = 3 +_base6.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.')] +azure_records.append(_base6) + +_base7 = _AzureRecord('TestAzure', octo_records[7]) +_base7.zone_name = 'unit.tests' +_base7.relative_record_set_name = '@' +_base7.record_type = 'NS' +_base7.params['ttl'] = 4 +_base7.params['ns_records'] = [NsRecord('ns1.unit.tests.'), + NsRecord('ns2.unit.tests.')] +azure_records.append(_base7) + +_base8 = _AzureRecord('TestAzure', octo_records[8]) +_base8.zone_name = 'unit.tests' +_base8.relative_record_set_name = 'foo' +_base8.record_type = 'NS' +_base8.params['ttl'] = 5 +_base8.params['ns_records'] = [NsRecord('ns1.unit.tests.')] +azure_records.append(_base8) + +_base9 = _AzureRecord('TestAzure', octo_records[9]) +_base9.zone_name = 'unit.tests' +_base9.relative_record_set_name = '_srv._tcp' +_base9.record_type = 'SRV' +_base9.params['ttl'] = 6 +_base9.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'), + SrvRecord(12, 30, 30, 'foo-2.unit.tests.')] +azure_records.append(_base9) + +_base10 = _AzureRecord('TestAzure', octo_records[10]) +_base10.zone_name = 'unit.tests' +_base10.relative_record_set_name = '_srv2._tcp' +_base10.record_type = 'SRV' +_base10.params['ttl'] = 7 +_base10.params['srv_records'] = [SrvRecord(12, 17, 1, 'srvfoo.unit.tests.')] +azure_records.append(_base10) class Test_AzureRecord(TestCase): - zone = Zone(name='unit.tests.', sub_zones=[]) - octo_records = [] - octo_records.append(Record.new(zone, '', { - 'ttl': 0, - 'type': 'A', - 'values': ['1.2.3.4', '10.10.10.10'] - })) - octo_records.append(Record.new(zone, 'a', { - 'ttl': 1, - 'type': 'A', - 'values': ['1.2.3.4', '1.1.1.1'], - })) - octo_records.append(Record.new(zone, 'aa', { - 'ttl': 9001, - 'type': 'A', - 'values': ['1.2.4.3'] - })) - octo_records.append(Record.new(zone, 'aaa', { - 'ttl': 2, - 'type': 'A', - 'values': ['1.1.1.3'] - })) - octo_records.append(Record.new(zone, 'cname', { - 'ttl': 3, - 'type': 'CNAME', - 'value': 'a.unit.tests.', - })) - octo_records.append(Record.new(zone, '', { - 'ttl': 3, - 'type': 'MX', - 'values': [{ - 'priority': 10, - 'value': 'mx1.unit.tests.', - }, { - 'priority': 20, - 'value': 'mx2.unit.tests.', - }] - })) - octo_records.append(Record.new(zone, '', { - 'ttl': 4, - 'type': 'NS', - 'values': ['ns1.unit.tests.', 'ns2.unit.tests.'], - })) - octo_records.append(Record.new(zone, '', { - 'ttl': 5, - 'type': 'NS', - 'value': 'ns1.unit.tests.', - })) - octo_records.append(Record.new(zone, '_srv._tcp', { - 'ttl': 6, - 'type': 'SRV', - 'values': [{ - 'priority': 10, - 'weight': 20, - 'port': 30, - 'target': 'foo-1.unit.tests.', - }, { - 'priority': 12, - 'weight': 30, - 'port': 30, - 'target': 'foo-2.unit.tests.', - }] - })) - - azure_records = [] - _base0 = _AzureRecord('TestAzure', octo_records[0]) - _base0.zone_name = 'unit.tests' - _base0.relative_record_set_name = '@' - _base0.record_type = 'A' - _base0.params['ttl'] = 0 - _base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')] - azure_records.append(_base0) - - _base1 = _AzureRecord('TestAzure', octo_records[1]) - _base1.zone_name = 'unit.tests' - _base1.relative_record_set_name = 'a' - _base1.record_type = 'A' - _base1.params['ttl'] = 1 - _base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')] - azure_records.append(_base1) - - _base2 = _AzureRecord('TestAzure', octo_records[2]) - _base2.zone_name = 'unit.tests' - _base2.relative_record_set_name = 'aa' - _base2.record_type = 'A' - _base2.params['ttl'] = 9001 - _base2.params['arecords'] = ARecord('1.2.4.3') - azure_records.append(_base2) - - _base3 = _AzureRecord('TestAzure', octo_records[3]) - _base3.zone_name = 'unit.tests' - _base3.relative_record_set_name = 'aaa' - _base3.record_type = 'A' - _base3.params['ttl'] = 2 - _base3.params['arecords'] = ARecord('1.1.1.3') - azure_records.append(_base3) - - _base4 = _AzureRecord('TestAzure', octo_records[4]) - _base4.zone_name = 'unit.tests' - _base4.relative_record_set_name = 'cname' - _base4.record_type = 'CNAME' - _base4.params['ttl'] = 3 - _base4.params['cname_record'] = CnameRecord('a.unit.tests.') - azure_records.append(_base4) - - _base5 = _AzureRecord('TestAzure', octo_records[5]) - _base5.zone_name = 'unit.tests' - _base5.relative_record_set_name = '@' - _base5.record_type = 'MX' - _base5.params['ttl'] = 3 - _base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'), - MxRecord(20, 'mx2.unit.tests.')] - azure_records.append(_base5) - - _base6 = _AzureRecord('TestAzure', octo_records[6]) - _base6.zone_name = 'unit.tests' - _base6.relative_record_set_name = '@' - _base6.record_type = 'NS' - _base6.params['ttl'] = 4 - _base6.params['ns_records'] = [NsRecord('ns1.unit.tests.'), - NsRecord('ns2.unit.tests.')] - azure_records.append(_base6) - - _base7 = _AzureRecord('TestAzure', octo_records[7]) - _base7.zone_name = 'unit.tests' - _base7.relative_record_set_name = '@' - _base7.record_type = 'NS' - _base7.params['ttl'] = 5 - _base7.params['ns_records'] = [NsRecord('ns1.unit.tests.')] - azure_records.append(_base7) - - _base8 = _AzureRecord('TestAzure', octo_records[8]) - _base8.zone_name = 'unit.tests' - _base8.relative_record_set_name = '_srv._tcp' - _base8.record_type = 'SRV' - _base8.params['ttl'] = 6 - _base8.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'), - SrvRecord(12, 30, 30, 'foo-2.unit.tests.')] - azure_records.append(_base8) - def test_azure_record(self): - assert(len(self.azure_records) == len(self.octo_records)) - for i in range(len(self.azure_records)): - octo = _AzureRecord('TestAzure', self.octo_records[i]) - assert(self.azure_records[i]._equals(octo)) + assert(len(azure_records) == len(octo_records)) + for i in range(len(azure_records)): + octo = _AzureRecord('TestAzure', octo_records[i]) + assert(azure_records[i]._equals(octo)) + string = str(azure_records[i]) + assert(('Ttl: ' in string)) + + +class TestValidatePeriod(TestCase): + def test_validate_per(self): + for expected, test in [['a.', 'a'], + ['a.', 'a.'], + ['foo.bar.', 'foo.bar.'], + ['foo.bar.', 'foo.bar']]: + self.assertEquals(expected, _validate_per(test)) class TestAzureDnsProvider(TestCase): - def test_populate(self): - pass # placeholder + def _provider(self): + return self._get_provider('mock_spc', 'mock_dns_client') + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def _get_provider(self, mock_spc, mock_dns_client): + '''Returns a mock AzureProvider object to use in testing. + + :param mock_spc: placeholder + :type mock_spc: str + :param mock_dns_client: placeholder + :type mock_dns_client: str + + :type return: AzureProvider + ''' + return AzureProvider('mock_id', 'mock_client', 'mock_key', + 'mock_directory', 'mock_sub', 'mock_rg') + + def test_populate_records(self): + provider = self._get_provider() + + rs = [] + rs.append(RecordSet(name='a1', ttl=0, type='A', + arecords=[ARecord('1.1.1.1')])) + rs.append(RecordSet(name='a2', ttl=1, type='A', + arecords=[ARecord('1.1.1.1'), + ARecord('2.2.2.2')])) + rs.append(RecordSet(name='aaaa1', ttl=2, type='AAAA', + aaaa_records=[AaaaRecord('1:1ec:1::1')])) + rs.append(RecordSet(name='aaaa2', ttl=3, type='AAAA', + aaaa_records=[AaaaRecord('1:1ec:1::1'), + AaaaRecord('1:1ec:1::2')])) + rs.append(RecordSet(name='cname1', ttl=4, type='CNAME', + cname_record=CnameRecord('cname.unit.test.'))) + rs.append(RecordSet(name='cname2', ttl=5, type='CNAME', + cname_record=None)) + rs.append(RecordSet(name='mx1', ttl=6, type='MX', + mx_records=[MxRecord(10, 'mx1.unit.test.')])) + rs.append(RecordSet(name='mx2', ttl=7, type='MX', + mx_records=[MxRecord(10, 'mx1.unit.test.'), + MxRecord(11, 'mx2.unit.test.')])) + rs.append(RecordSet(name='ns1', ttl=8, type='NS', + ns_records=[NsRecord('ns1.unit.test.')])) + rs.append(RecordSet(name='ns2', ttl=9, type='NS', + ns_records=[NsRecord('ns1.unit.test.'), + NsRecord('ns2.unit.test.')])) + rs.append(RecordSet(name='ptr1', ttl=10, type='PTR', + ptr_records=[PtrRecord('ptr1.unit.test.')])) + rs.append(RecordSet(name='ptr2', ttl=11, type='PTR', + ptr_records=[PtrRecord('ptr1.unit.test.'), + PtrRecord('ptr2.unit.test.')])) + rs.append(RecordSet(name='_srv1._tcp', ttl=12, type='SRV', + srv_records=[SrvRecord(1, 2, 3, '1unit.tests.')])) + rs.append(RecordSet(name='_srv2._tcp', ttl=13, type='SRV', + srv_records=[SrvRecord(1, 2, 3, '1unit.tests.'), + SrvRecord(4, 5, 6, '2unit.tests.')])) + rs.append(RecordSet(name='txt1', ttl=14, type='TXT', + txt_records=[TxtRecord('sample text1')])) + rs.append(RecordSet(name='txt2', ttl=15, type='TXT', + txt_records=[TxtRecord('sample text1'), + TxtRecord('sample text2')])) + rs.append(RecordSet(name='', ttl=16, type='SOA', + soa_record=[SoaRecord()])) + + record_list = provider._dns_client.record_sets.list_by_dns_zone + record_list.return_value = rs + + provider.populate(zone) + + self.assertEquals(len(zone.records), 16) + + def test_populate_zone(self): + provider = self._get_provider() + + zone_list = provider._dns_client.zones.list_by_resource_group + zone_list.return_value = [AzureZone(location='global'), + AzureZone(location='global')] + + provider._populate_zones() + + self.assertEquals(len(provider._azure_zones), 1) + + def test_bad_zone_response(self): + provider = self._get_provider() + + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), 'Azure Error') + trip = False + try: + provider._check_zone('unit.test', create=False) + except CloudError: + trip = True + self.assertEquals(trip, True) + + def test_apply(self): + provider = self._get_provider() + + changes = [] + deletes = [] + for i in octo_records: + changes.append(Create(i)) + deletes.append(Delete(i)) + + self.assertEquals(11, provider.apply(Plan(None, zone, changes))) + self.assertEquals(11, provider.apply(Plan(zone, zone, deletes))) + + def test_create_zone(self): + provider = self._get_provider() + + changes = [] + for i in octo_records: + changes.append(Create(i)) + desired = Zone('unit2.test.', []) + + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), 'Azure Error') + + self.assertEquals(11, provider.apply(Plan(None, desired, changes))) From 824cf4e98c3cd9e334fe9def9402dfefdb4abc16 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 30 Jun 2017 17:12:18 -0700 Subject: [PATCH 29/84] Changed code as per PR review. Only major change is refactoring _check_zones. Many more comments --- .gitignore | 2 - doit.txt | 4 - octodns/provider/azuredns.py | 146 ++++++++++++++++-------- tests/test_octodns_provider_azuredns.py | 43 ++++++- 4 files changed, 138 insertions(+), 57 deletions(-) delete mode 100755 doit.txt diff --git a/.gitignore b/.gitignore index 5a3f3f9..c45a684 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,3 @@ output/ tmp/ build/ config/ -./rb.txt -./doit.txt diff --git a/doit.txt b/doit.txt deleted file mode 100755 index 1ff2da6..0000000 --- a/doit.txt +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -#script to rebuild octodns quickly - -octodns-sync --config-file=./config/production.yaml --doit diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index cf0e2d5..91e298c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -9,6 +9,7 @@ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone +from msrestazure.azure_exceptions import CloudError from functools import reduce import sys @@ -18,7 +19,8 @@ from .base import BaseProvider class _AzureRecord(object): - ''' Wrapper for OctoDNS record. + '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. + azuredns.py: class: octodns.provider.azuredns._AzureRecord An _AzureRecord is easily accessible to Azure DNS Management library @@ -27,7 +29,19 @@ class _AzureRecord(object): ''' def __init__(self, resource_group, record, delete=False): - ''' + '''Contructor for _AzureRecord. + + Notes on Azure records: An Azure record set has the form + RecordSet(name=<...>, type=<...>, arecords=[...], aaaa_records, ..) + When constructing an azure record as done in self._apply_Create, + the argument parameters for an A record would be + parameters={'ttl': , 'arecords': [ARecord(),]}. + As another example for CNAME record: + parameters={'ttl': , 'cname_record': CnameRecord()}. + + Below, key_name and class_name are the dictionary key and Azure + Record class respectively. + :param resource_group: The name of resource group in Azure :type resource_group: str :param record: An OctoDNS record @@ -39,15 +53,18 @@ class _AzureRecord(object): :type return: _AzureRecord ''' self.resource_group = resource_group - self.zone_name = record.zone.name[0:len(record.zone.name) - 1] + self.zone_name = record.zone.name[:len(record.zone.name) - 1] self.relative_record_set_name = record.name or '@' self.record_type = record._type if delete: return + # Refer to function docstring for key_name and class_name. format_u_s = '' if record._type == 'A' else '_' key_name = '{}{}records'.format(self.record_type, format_u_s).lower() + if record._type == 'CNAME': + key_name = key_name[:len(key_name) - 1] class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) @@ -56,8 +73,10 @@ class _AzureRecord(object): self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): - return {key_name: [azure_class(v) for v in data['values']]} \ - if 'values' in data else {key_name: [azure_class(data['value'])]} + if 'values' in data: + return {key_name: [azure_class(v) for v in data['values']]} + else: # Else there is a singular data point keyed by 'value'. + return {key_name: [azure_class(data['value'])]} _params_for_A = _params _params_for_AAAA = _params @@ -73,7 +92,7 @@ class _AzureRecord(object): vals['weight'], vals['port'], vals['target'])) - else: # single value at key 'value' + else: # Else there is a singular data point keyed by 'value'. params.append(azure_class(data['value']['priority'], data['value']['weight'], data['value']['port'], @@ -86,7 +105,7 @@ class _AzureRecord(object): for vals in data['values']: params.append(azure_class(vals['preference'], vals['exchange'])) - else: # single value at key 'value' + else: # Else there is a singular data point keyed by 'value'. params.append(azure_class(data['value']['preference'], data['value']['exchange'])) return {key_name: params} @@ -141,10 +160,14 @@ class _AzureRecord(object): return string -def _validate_per(string): +def _check_endswith_dot(string): return string if string.endswith('.') else string + '.' +def _parse_azure_type(string): + return string.split('/')[len(string.split('/')) - 1] + + class AzureProvider(BaseProvider): ''' Azure DNS Provider @@ -155,21 +178,44 @@ class AzureProvider(BaseProvider): # includes using a Service Principal: # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ # resource-group-create-service-principal-portal - # The Azure Active Directory Application ID (aka client ID) req: + # The Azure Active Directory Application ID (aka client ID): client_id: - # Authentication Key Value req: + # Authentication Key Value: (note this should be secret) key: - # Directory ID (referred to tenant ID) req: + # Directory ID (aka tenant ID): directory_id: - # Subscription ID req: + # Subscription ID: sub_id: - # Resource Group name req: + # Resource Group name: resource_group: + # All are required to authenticate. - TODO: change config file to use env vars instead of hard-coded keys + Example config file with variables: + " + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config (example path to directory of zone files) + azuredns: + class: octodns.provider.azuredns.AzureProvider + client_id: env/AZURE_APPLICATION_ID + key: env/AZURE_AUTHENICATION_KEY + directory_id: env/AZURE_DIRECTORY_ID + sub_id: env/AZURE_SUBSCRIPTION_ID + resource_group: 'TestResource1' - personal notes: testing: test authentication vars located in - /home/t-hehwan/vars.txt + zones: + example.com.: + sources: + - config + targets: + - azuredns + " + The first four variables above can be hidden in environment variables + and octoDNS will automatically search for them in the shell. It is + possible to also hard-code into the config file. resource_group can + also be an environment variable but might likely change. ''' SUPPORTS_GEO = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) @@ -214,19 +260,24 @@ class AzureProvider(BaseProvider): self._dns_client.zones.get(self._resource_group, name) self._azure_zones.add(name) return name - except: - if create: - self.log.debug('_check_zone:no matching zone; creating %s', - name) - create_zone = self._dns_client.zones.create_or_update - create_zone(self._resource_group, name, Zone('global')) - return name - else: - raise + except CloudError as err: + msg = 'The Resource \'Microsoft.Network/dnszones/{}\''.format(name) + msg += ' under resource group \'{}\''.format(self._resource_group) + msg += ' was not found.' + if msg == err.message: + # Then the only error is that the zone doesn't currently exist + if create: + self.log.debug('_check_zone:no matching zone; creating %s', + name) + create_zone = self._dns_client.zones.create_or_update + create_zone(self._resource_group, name, Zone('global')) + return name + else: + return + raise def populate(self, zone, target=False, lenient=False): - ''' - Required function of manager.py. + '''Required function of manager.py to collect records from zone. Special notes for Azure. Azure zone names omit final '.' @@ -249,26 +300,28 @@ class AzureProvider(BaseProvider): :type return: void ''' - zone_name = zone.name[0:len(zone.name) - 1] - self.log.debug('populate: name=%s', zone_name) + self.log.debug('populate: name=%s', zone.name) before = len(zone.records) + zone_name = zone.name[:len(zone.name) - 1] self._populate_zones() self._check_zone(zone_name) _records = set() records = self._dns_client.record_sets.list_by_dns_zone - for azrecord in records(self._resource_group, zone_name): - if azrecord.type in self.SUPPORTS: - _records.add(azrecord) - for azrecord in _records: - record_name = azrecord.name if azrecord.name != '@' else '' - data = getattr(self, '_data_for_{}'.format(azrecord.type)) - data = data(azrecord) - data['type'] = azrecord.type - data['ttl'] = azrecord.ttl - record = Record.new(zone, record_name, data, source=self) - zone.add_record(record) + if self._check_zone(zone_name): + for azrecord in records(self._resource_group, zone_name): + if _parse_azure_type(azrecord.type) in self.SUPPORTS: + _records.add(azrecord) + for azrecord in _records: + record_name = azrecord.name if azrecord.name != '@' else '' + typ = _parse_azure_type(azrecord.type) + data = getattr(self, '_data_for_{}'.format(typ)) + data = data(azrecord) + data['type'] = typ + data['ttl'] = azrecord.ttl + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) self.log.info('populate: found %s records', len(zone.records) - before) @@ -293,13 +346,14 @@ class AzureProvider(BaseProvider): records. Refer to population comment. ''' try: - return {'value': _validate_per(azrecord.cname_record.cname)} + return {'value': _check_endswith_dot(azrecord.cname_record.cname)} except: return {'value': '.'} def _data_for_PTR(self, azrecord): try: - return {'value': _validate_per(azrecord.ptr_records[0].ptdrname)} + ptdrname = azrecord.ptr_records[0].ptdrname + return {'value': _check_endswith_dot(ptdrname)} except: return {'value': '.'} @@ -315,7 +369,7 @@ class AzureProvider(BaseProvider): def _data_for_NS(self, azrecord): vals = [ar.nsdname for ar in azrecord.ns_records] - return {'values': [_validate_per(val) for val in vals]} + return {'values': [_check_endswith_dot(val) for val in vals]} def _apply_Create(self, change): '''A record from change must be created. @@ -334,7 +388,7 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - self.log.debug('* Success Create/Update: {}'.format(ar)) + print('* Success Create/Update: {}'.format(ar), file=sys.stderr) _apply_Update = _apply_Create @@ -345,7 +399,7 @@ class AzureProvider(BaseProvider): delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - self.log.debug('* Success Delete: {}'.format(ar)) + print('* Success Delete: {}'.format(ar), file=sys.stderr) def _apply(self, plan): ''' @@ -361,7 +415,7 @@ class AzureProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - azure_zone_name = desired.name[0:len(desired.name) - 1] + azure_zone_name = desired.name[:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) for change in changes: diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index edb2db2..abce5aa 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from octodns.record import Create, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ - _validate_per + _check_endswith_dot, _parse_azure_type from octodns.zone import Zone from octodns.provider.base import Plan @@ -195,13 +195,22 @@ class Test_AzureRecord(TestCase): assert(('Ttl: ' in string)) -class TestValidatePeriod(TestCase): - def test_validate_per(self): +class Test_ParseAzureType(TestCase): + def test_parse_azure_type(self): + for expected, test in [['A', 'Microsoft.Network/dnszones/A'], + ['AAAA', 'Microsoft.Network/dnszones/AAAA'], + ['NS', 'Microsoft.Network/dnszones/NS'], + ['MX', 'Microsoft.Network/dnszones/MX']]: + self.assertEquals(expected, _parse_azure_type(test)) + + +class Test_CheckEndswithDot(TestCase): + def test_check_endswith_dot(self): for expected, test in [['a.', 'a'], ['a.', 'a.'], ['foo.bar.', 'foo.bar.'], ['foo.bar.', 'foo.bar']]: - self.assertEquals(expected, _validate_per(test)) + self.assertEquals(expected, _check_endswith_dot(test)) class TestAzureDnsProvider(TestCase): @@ -319,7 +328,31 @@ class TestAzureDnsProvider(TestCase): changes.append(Create(i)) desired = Zone('unit2.test.', []) + err_msg = 'The Resource \'Microsoft.Network/dnszones/unit2.test\' ' + err_msg += 'under resource group \'mock_rg\' was not found.' _get = provider._dns_client.zones.get - _get.side_effect = CloudError(Mock(status=404), 'Azure Error') + _get.side_effect = CloudError(Mock(status=404), err_msg) self.assertEquals(11, provider.apply(Plan(None, desired, changes))) + + def test_check_zone_no_create(self): + provider = self._get_provider() + + rs = [] + rs.append(RecordSet(name='a1', ttl=0, type='A', + arecords=[ARecord('1.1.1.1')])) + rs.append(RecordSet(name='a2', ttl=1, type='A', + arecords=[ARecord('1.1.1.1'), + ARecord('2.2.2.2')])) + + record_list = provider._dns_client.record_sets.list_by_dns_zone + record_list.return_value = rs + + err_msg = 'The Resource \'Microsoft.Network/dnszones/unit3.test\' ' + err_msg += 'under resource group \'mock_rg\' was not found.' + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), err_msg) + + provider.populate(Zone('unit3.test.', [])) + + self.assertEquals(len(zone.records), 0) From ec4261e7da97bf8711c830965f18f7da3e725439 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 30 Jun 2017 17:29:16 -0700 Subject: [PATCH 30/84] Fixed typed in _data_for_PTR and amended test case to check for it --- octodns/provider/azuredns.py | 4 ++-- tests/test_octodns_provider_azuredns.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 91e298c..4fe5184 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -352,8 +352,8 @@ class AzureProvider(BaseProvider): def _data_for_PTR(self, azrecord): try: - ptdrname = azrecord.ptr_records[0].ptdrname - return {'value': _check_endswith_dot(ptdrname)} + ptrdname = azrecord.ptr_records[0].ptrdname + return {'value': _check_endswith_dot(ptrdname)} except: return {'value': '.'} diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index abce5aa..5ffa055 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -263,8 +263,7 @@ class TestAzureDnsProvider(TestCase): rs.append(RecordSet(name='ptr1', ttl=10, type='PTR', ptr_records=[PtrRecord('ptr1.unit.test.')])) rs.append(RecordSet(name='ptr2', ttl=11, type='PTR', - ptr_records=[PtrRecord('ptr1.unit.test.'), - PtrRecord('ptr2.unit.test.')])) + ptr_records=[PtrRecord(None)])) rs.append(RecordSet(name='_srv1._tcp', ttl=12, type='SRV', srv_records=[SrvRecord(1, 2, 3, '1unit.tests.')])) rs.append(RecordSet(name='_srv2._tcp', ttl=13, type='SRV', From d9806e851f2ef7a2a8717cda1fa49cd8687e67e7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 2 Jul 2017 10:45:58 -0700 Subject: [PATCH 31/84] NS1 RateLimitException, just sleep for e.period --- octodns/provider/ns1.py | 18 ++++++++---------- tests/test_octodns_provider_ns1.py | 8 ++++---- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 0f3db1f..bca6118 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -28,13 +28,11 @@ class Ns1Provider(BaseProvider): ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' - def __init__(self, id, api_key, rate_limit_delay=1, *args, **kwargs): + def __init__(self, id, api_key, *args, **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) - self.log.debug('__init__: id=%s, api_key=***, rate_limit_delay=%d', id, - rate_limit_delay) + self.log.debug('__init__: id=%s, api_key=***', id) super(Ns1Provider, self).__init__(id, *args, **kwargs) self._client = NSONE(apiKey=api_key) - self.rate_limit_delay = rate_limit_delay def _data_for_A(self, _type, record): return { @@ -177,10 +175,10 @@ class Ns1Provider(BaseProvider): meth = getattr(nsone_zone, 'add_{}'.format(_type)) try: meth(name, **params) - except RateLimitException: + except RateLimitException as e: self.log.warn('_apply_Create: rate limit encountered, pausing ' 'and trying again') - sleep(self.rate_limit_delay) + sleep(e.period) meth(name, **params) def _apply_Update(self, nsone_zone, change): @@ -192,10 +190,10 @@ class Ns1Provider(BaseProvider): params = getattr(self, '_params_for_{}'.format(_type))(new) try: record.update(**params) - except RateLimitException: + except RateLimitException as e: self.log.warn('_apply_Update: rate limit encountered, pausing ' 'and trying again') - sleep(self.rate_limit_delay) + sleep(e.period) record.update(**params) def _apply_Delete(self, nsone_zone, change): @@ -205,10 +203,10 @@ class Ns1Provider(BaseProvider): record = nsone_zone.loadRecord(name, _type) try: record.delete() - except RateLimitException: + except RateLimitException as e: self.log.warn('_apply_Delete: rate limit encountered, pausing ' 'and trying again') - sleep(self.rate_limit_delay) + sleep(e.period) record.delete() def _apply(self, plan): diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 5e53cfd..0398459 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -193,7 +193,7 @@ class TestNs1Provider(TestCase): @patch('nsone.NSONE.createZone') @patch('nsone.NSONE.loadZone') def test_sync(self, load_mock, create_mock): - provider = Ns1Provider('test', 'api-key', rate_limit_delay=0) + provider = Ns1Provider('test', 'api-key') desired = Zone('unit.tests.', []) desired.records.update(self.expected) @@ -231,7 +231,7 @@ class TestNs1Provider(TestCase): mock_zone = Mock() mock_zone.add_SRV = Mock() mock_zone.add_SRV.side_effect = [ - RateLimitException('boo'), + RateLimitException('boo', period=0), None, ] create_mock.side_effect = [mock_zone] @@ -259,11 +259,11 @@ class TestNs1Provider(TestCase): # trigger rate limit handling mock_record = Mock() mock_record.update.side_effect = [ - RateLimitException('one'), + RateLimitException('one', period=0), None, ] mock_record.delete.side_effect = [ - RateLimitException('two'), + RateLimitException('two', period=0), None, ] nsone_zone.loadRecord.side_effect = [mock_record, mock_record] From 06fb57855015a372b60aeea5342023247d9fd8a2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 2 Jul 2017 10:47:13 -0700 Subject: [PATCH 32/84] Include sleep duration in ns1 RateLimitException log --- octodns/provider/ns1.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index bca6118..65db64c 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -177,7 +177,7 @@ class Ns1Provider(BaseProvider): meth(name, **params) except RateLimitException as e: self.log.warn('_apply_Create: rate limit encountered, pausing ' - 'and trying again') + 'for %ds and trying again', e.period) sleep(e.period) meth(name, **params) @@ -192,7 +192,7 @@ class Ns1Provider(BaseProvider): record.update(**params) except RateLimitException as e: self.log.warn('_apply_Update: rate limit encountered, pausing ' - 'and trying again') + 'for %ds and trying again', e.period) sleep(e.period) record.update(**params) @@ -205,7 +205,7 @@ class Ns1Provider(BaseProvider): record.delete() except RateLimitException as e: self.log.warn('_apply_Delete: rate limit encountered, pausing ' - 'and trying again') + 'for %ds and trying again', e.period) sleep(e.period) record.delete() From 908698da492bd0e71c111bae4d59669ea4e83827 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 2 Jul 2017 18:23:45 -0700 Subject: [PATCH 33/84] Fix major performance issue with add_record O(N^2) Before, 1-2k record took ~10s and more than that was just painful, 5k took forever. This records things to keep a dict of nodes with a set of records so that we can quickly "jump" to the point we're looking for without having to search. 10k records now takes ~5s. --- octodns/__init__.py | 4 +- octodns/zone.py | 47 +++++++++++++++-------- tests/test_octodns_provider_cloudflare.py | 2 +- tests/test_octodns_provider_dnsimple.py | 2 +- tests/test_octodns_provider_ns1.py | 3 +- tests/test_octodns_provider_powerdns.py | 2 +- tests/test_octodns_provider_route53.py | 12 +++--- tests/test_octodns_zone.py | 2 +- 8 files changed, 44 insertions(+), 30 deletions(-) diff --git a/octodns/__init__.py b/octodns/__init__.py index b6287e5..94ef299 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -1,6 +1,4 @@ -''' -OctoDNS: DNS as code - Tools for managing DNS across multiple providers -''' +'OctoDNS: DNS as code - Tools for managing DNS across multiple providers' from __future__ import absolute_import, division, print_function, \ unicode_literals diff --git a/octodns/zone.py b/octodns/zone.py index 9d405bb..74e5d9e 100644 --- a/octodns/zone.py +++ b/octodns/zone.py @@ -5,6 +5,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +from collections import defaultdict from logging import getLogger import re @@ -39,13 +40,19 @@ class Zone(object): # Force everyting to lowercase just to be safe self.name = str(name).lower() if name else name self.sub_zones = sub_zones - self.records = set() + # We're grouping by node, it allows us to efficently search for + # duplicates and detect when CNAMEs co-exist with other records + self._records = defaultdict(set) # optional leading . to match empty hostname # optional trailing . b/c some sources don't have it on their fqdn self._name_re = re.compile('\.?{}?$'.format(name)) self.log.debug('__init__: zone=%s, sub_zones=%s', self, sub_zones) + @property + def records(self): + return set([r for _, node in self._records.items() for r in node]) + def hostname_from_fqdn(self, fqdn): return self._name_re.sub('', fqdn) @@ -53,9 +60,6 @@ class Zone(object): name = record.name last = name.split('.')[-1] - if replace and record in self.records: - self.records.remove(record) - if last in self.sub_zones: if name != last: # it's a record for something under a sub-zone @@ -67,19 +71,30 @@ class Zone(object): raise SubzoneRecordException('Record {} a managed sub-zone ' 'and not of type NS' .format(record.fqdn)) - # TODO: this is pretty inefficent - for existing in self.records: - if record == existing: - raise DuplicateRecordException('Duplicate record {}, type {}' - .format(record.fqdn, - record._type)) - elif name == existing.name and (record._type == 'CNAME' or - existing._type == 'CNAME'): - raise InvalidNodeException('Invalid state, CNAME at {} ' - 'cannot coexist with other records' - .format(record.fqdn)) - self.records.add(record) + if replace: + # will remove it if it exists + self._records[name].discard(record) + + node = self._records[name] + if record in node: + # We already have a record at this node of this type + raise DuplicateRecordException('Duplicate record {}, type {}' + .format(record.fqdn, + record._type)) + elif ((record._type == 'CNAME' and len(node) > 0) or + ('CNAME' in map(lambda r: r._type, node))): + # We're adding a CNAME to existing records or adding to an existing + # CNAME + raise InvalidNodeException('Invalid state, CNAME at {} cannot ' + 'coexist with other records' + .format(record.fqdn)) + + node.add(record) + + def _remove_record(self, record): + 'Only for use in tests' + self._records[record.name].discard(record) def changes(self, desired, target): self.log.debug('changes: zone=%s, target=%s', self, target) diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 3f652a1..5dcae30 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -33,7 +33,7 @@ class TestCloudflareProvider(TestCase): })) for record in list(expected.records): if record.name == 'sub' and record._type == 'NS': - expected.records.remove(record) + expected._remove_record(record) break empty = {'result': [], 'result_info': {'count': 0, 'per_page': 0}} diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index 1f62bfd..aed1e8b 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -33,7 +33,7 @@ class TestDnsimpleProvider(TestCase): })) for record in list(expected.records): if record.name == 'sub' and record._type == 'NS': - expected.records.remove(record) + expected._remove_record(record) break def test_populate(self): diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 0398459..ce1353b 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -196,7 +196,8 @@ class TestNs1Provider(TestCase): provider = Ns1Provider('test', 'api-key') desired = Zone('unit.tests.', []) - desired.records.update(self.expected) + for r in self.expected: + desired.add_record(r) plan = provider.plan(desired) # everything except the root NS diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 01e7d83..5fcd80a 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -253,7 +253,7 @@ class TestPowerDnsProvider(TestCase): plan = provider.plan(expected) self.assertFalse(plan) # remove it now that we don't need the unrelated change any longer - expected.records.remove(unrelated_record) + expected._remove_record(unrelated_record) # ttl diff with requests_mock() as mock: diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index cad58f8..be624ff 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -372,11 +372,11 @@ class TestRoute53Provider(TestCase): # Delete by monkey patching in a populate that includes an extra record def add_extra_populate(existing, target, lenient): for record in self.expected.records: - existing.records.add(record) + existing.add_record(record) record = Record.new(existing, 'extra', {'ttl': 99, 'type': 'A', 'values': ['9.9.9.9']}) - existing.records.add(record) + existing.add_record(record) provider.populate = add_extra_populate change_resource_record_sets_params = { @@ -409,7 +409,7 @@ class TestRoute53Provider(TestCase): def mod_geo_populate(existing, target, lenient): for record in self.expected.records: if record._type != 'A' or not record.geo: - existing.records.add(record) + existing.add_record(record) record = Record.new(existing, '', { 'ttl': 61, 'type': 'A', @@ -420,7 +420,7 @@ class TestRoute53Provider(TestCase): 'NA-US-KY': ['7.2.3.4'] } }) - existing.records.add(record) + existing.add_record(record) provider.populate = mod_geo_populate change_resource_record_sets_params = { @@ -505,7 +505,7 @@ class TestRoute53Provider(TestCase): def mod_add_geo_populate(existing, target, lenient): for record in self.expected.records: if record._type != 'A' or record.geo: - existing.records.add(record) + existing.add_record(record) record = Record.new(existing, 'simple', { 'ttl': 61, 'type': 'A', @@ -514,7 +514,7 @@ class TestRoute53Provider(TestCase): 'OC': ['3.2.3.4', '4.2.3.4'], } }) - existing.records.add(record) + existing.add_record(record) provider.populate = mod_add_geo_populate change_resource_record_sets_params = { diff --git a/tests/test_octodns_zone.py b/tests/test_octodns_zone.py index f310397..8d75100 100644 --- a/tests/test_octodns_zone.py +++ b/tests/test_octodns_zone.py @@ -77,7 +77,7 @@ class TestZone(TestCase): # add a record, delete a record -> [Delete, Create] c = ARecord(before, 'c', {'ttl': 42, 'value': '1.1.1.1'}) after.add_record(c) - after.records.remove(b) + after._remove_record(b) self.assertEquals(after.records, set([a, c])) changes = before.changes(after, target) self.assertEquals(2, len(changes)) From bdceac42beb485ed30417a1ae85db1f6e45673b2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 2 Jul 2017 18:40:58 -0700 Subject: [PATCH 34/84] Fix stacktraces on MainThreadExecutor --- octodns/manager.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index e6fe253..8439eb6 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from StringIO import StringIO -from concurrent.futures import Future, ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor from importlib import import_module from os import environ import logging @@ -38,6 +38,17 @@ class _AggregateTarget(object): return True +class MakeThreadFuture(object): + + def __init__(self, func, args, kwargs): + self.func = func + self.args = args + self.kwargs = kwargs + + def result(self): + return self.func(*self.args, **self.kwargs) + + class MainThreadExecutor(object): ''' Dummy executor that runs things on the main thread during the involcation @@ -48,13 +59,7 @@ class MainThreadExecutor(object): ''' def submit(self, func, *args, **kwargs): - future = Future() - try: - future.set_result(func(*args, **kwargs)) - except Exception as e: - # TODO: get right stacktrace here - future.set_exception(e) - return future + return MakeThreadFuture(func, args, kwargs) class Manager(object): From 9637218c8b544c397bcd5d433de47cafbfad973d Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Mon, 3 Jul 2017 16:10:48 +0200 Subject: [PATCH 35/84] Add lenient to abstract BaseSource signature --- octodns/source/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/source/base.py b/octodns/source/base.py index 2e2c5c2..4ace09f 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -20,7 +20,7 @@ class BaseSource(object): raise NotImplementedError('Abstract base class, SUPPORTS ' 'property missing') - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): ''' Loads all zones the provider knows about From cd9d7254f06b1f5d24524bae0e236367642ff899 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 3 Jul 2017 11:03:23 -0700 Subject: [PATCH 36/84] Fixed stray prints and assert errors. Added versions of required azure libraries --- octodns/provider/azuredns.py | 30 +++++++++++++++++++++--------- requirements.txt | 3 +++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4fe5184..0abddb0 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -7,17 +7,30 @@ from __future__ import absolute_import, division, print_function, \ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient -from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ - SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone from msrestazure.azure_exceptions import CloudError -from functools import reduce -import sys +# Imports are used: 'self.params(record.data, key_name, eval(class_name))' +# To pass pyflakes import statement tests. +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord +from azure.mgmt.dns.models import Zone + import logging +from functools import reduce from ..record import Record from .base import BaseProvider +ARecord +AaaaRecord +CnameRecord +MxRecord +SrvRecord +NsRecord +PtrRecord +TxtRecord + + class _AzureRecord(object): '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. @@ -65,8 +78,7 @@ class _AzureRecord(object): key_name = '{}{}records'.format(self.record_type, format_u_s).lower() if record._type == 'CNAME': key_name = key_name[:len(key_name) - 1] - class_name = '{}'.format(self.record_type).capitalize() + \ - 'Record'.format(self.record_type) + class_name = '{}'.format(self.record_type).capitalize() + 'Record' self.params = getattr(self, '_params_for_{}'.format(record._type)) self.params = self.params(record.data, key_name, eval(class_name)) @@ -111,7 +123,7 @@ class _AzureRecord(object): return {key_name: params} def _params_for_CNAME(self, data, key_name, azure_class): - return {'cname_record': CnameRecord(data['value'])} + return {'cname_record': azure_class(data['value'])} def _equals(self, b): '''Checks whether two records are equal by comparing all fields. @@ -388,7 +400,7 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - print('* Success Create/Update: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Create/Update: {}'.format(ar)) _apply_Update = _apply_Create @@ -399,7 +411,7 @@ class AzureProvider(BaseProvider): delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - print('* Success Delete: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): ''' diff --git a/requirements.txt b/requirements.txt index b10ca4c..62c485a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,6 @@ python-dateutil==2.6.0 requests==2.13.0 s3transfer==0.1.10 six==1.10.0 +azure-mgmt-dns==1.0.1 +azure-common==1.1.6 +msrestazure==0.4.10 \ No newline at end of file From 348a6ca783af214632831961abd0a5aec8a2225a Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 3 Jul 2017 16:03:20 -0700 Subject: [PATCH 37/84] Changed to map types to Azure Records isntead of implicitly using eval --- octodns/provider/azuredns.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0abddb0..4ed783c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -9,11 +9,8 @@ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient from msrestazure.azure_exceptions import CloudError -# Imports are used: 'self.params(record.data, key_name, eval(class_name))' -# To pass pyflakes import statement tests. from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ - SrvRecord, NsRecord, PtrRecord, TxtRecord -from azure.mgmt.dns.models import Zone + SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone import logging from functools import reduce @@ -21,16 +18,6 @@ from ..record import Record from .base import BaseProvider -ARecord -AaaaRecord -CnameRecord -MxRecord -SrvRecord -NsRecord -PtrRecord -TxtRecord - - class _AzureRecord(object): '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. @@ -40,6 +27,16 @@ class _AzureRecord(object): functions and is used to wrap all relevant data to create a record in Azure. ''' + TYPE_MAP = { + 'A': ARecord, + 'AAAA': AaaaRecord, + 'CNAME': CnameRecord, + 'MX': MxRecord, + 'SRV': SrvRecord, + 'NS': NsRecord, + 'PTR': PtrRecord, + 'TXT': TxtRecord + } def __init__(self, resource_group, record, delete=False): '''Contructor for _AzureRecord. @@ -78,10 +75,10 @@ class _AzureRecord(object): key_name = '{}{}records'.format(self.record_type, format_u_s).lower() if record._type == 'CNAME': key_name = key_name[:len(key_name) - 1] - class_name = '{}'.format(self.record_type).capitalize() + 'Record' + azure_class = self.TYPE_MAP[self.record_type] self.params = getattr(self, '_params_for_{}'.format(record._type)) - self.params = self.params(record.data, key_name, eval(class_name)) + self.params = self.params(record.data, key_name, azure_class) self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): From 99578f328c18fc72f874da090952d7cd5e040e2f Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Wed, 5 Jul 2017 09:45:38 -0700 Subject: [PATCH 38/84] add azure to README. order reqs, change comments slightly, alphabetize functions --- README.md | 1 + octodns/provider/azuredns.py | 76 +++++++++++++------------ requirements.txt | 8 +-- tests/test_octodns_provider_azuredns.py | 1 - 4 files changed, 45 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 0e63e51..05cf979 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Record Support | GeoDNS Support | Notes | |--|--|--|--| +| [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CNAME, MX, NS, SPF, TXT | No | | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4ed783c..4433b1e 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -56,9 +56,8 @@ class _AzureRecord(object): :type resource_group: str :param record: An OctoDNS record :type record: ..record.Record - :param values: Parameters for a record. eg IP address, port, domain - name, etc. Values usually read from record.data - :type values: {'values': [...]} or {'value': [...]} + :param delete: If true, omit data parsing; not needed to delete + :type delete: bool :type return: _AzureRecord ''' @@ -93,6 +92,20 @@ class _AzureRecord(object): _params_for_PTR = _params _params_for_TXT = _params + def _params_for_CNAME(self, data, key_name, azure_class): + return {key_name: azure_class(data['value'])} + + def _params_for_MX(self, data, key_name, azure_class): + params = [] + if 'values' in data: + for vals in data['values']: + params.append(azure_class(vals['preference'], + vals['exchange'])) + else: # Else there is a singular data point keyed by 'value'. + params.append(azure_class(data['value']['preference'], + data['value']['exchange'])) + return {key_name: params} + def _params_for_SRV(self, data, key_name, azure_class): params = [] if 'values' in data: @@ -108,20 +121,6 @@ class _AzureRecord(object): data['value']['target'])) return {key_name: params} - def _params_for_MX(self, data, key_name, azure_class): - params = [] - if 'values' in data: - for vals in data['values']: - params.append(azure_class(vals['preference'], - vals['exchange'])) - else: # Else there is a singular data point keyed by 'value'. - params.append(azure_class(data['value']['preference'], - data['value']['exchange'])) - return {key_name: params} - - def _params_for_CNAME(self, data, key_name, azure_class): - return {'cname_record': azure_class(data['value'])} - def _equals(self, b): '''Checks whether two records are equal by comparing all fields. :param b: Another _AzureRecord object @@ -174,6 +173,13 @@ def _check_endswith_dot(string): def _parse_azure_type(string): + '''Converts string representing an Azure RecordSet type to usual type. + + :param string: the Azure type. eg: + :type string: str + + :type return: str + ''' return string.split('/')[len(string.split('/')) - 1] @@ -223,8 +229,7 @@ class AzureProvider(BaseProvider): " The first four variables above can be hidden in environment variables and octoDNS will automatically search for them in the shell. It is - possible to also hard-code into the config file. resource_group can - also be an environment variable but might likely change. + possible to also hard-code into the config file: eg, resource_group. ''' SUPPORTS_GEO = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) @@ -250,8 +255,8 @@ class AzureProvider(BaseProvider): self._azure_zones.add(zone.name) def _check_zone(self, name, create=False): - ''' - Checks whether a zone specified in a source exist in Azure server. + '''Checks whether a zone specified in a source exist in Azure server. + Note that Azure zones omit end '.' eg: contoso.com vs contoso.com. Returns the name if it exists. @@ -302,7 +307,7 @@ class AzureProvider(BaseProvider): :param zone: A dns zone :type zone: octodns.zone.Zone :param target: Checks if Azure is source or target of config. - Currently only supports as a target. Does not use. + Currently only supports as a target. Unused. :type target: bool :param lenient: Unused. Check octodns.manager for usage. :type lenient: bool @@ -340,10 +345,6 @@ class AzureProvider(BaseProvider): def _data_for_AAAA(self, azrecord): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} - def _data_for_TXT(self, azrecord): - return {'values': [reduce((lambda a, b: a + b), ar.value) - for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): '''Parsing data from Azure DNS Client record call :param azrecord: a return of a call to list azure records @@ -359,6 +360,15 @@ class AzureProvider(BaseProvider): except: return {'value': '.'} + def _data_for_MX(self, azrecord): + return {'values': [{'preference': ar.preference, + 'exchange': ar.exchange} + for ar in azrecord.mx_records]} + + def _data_for_NS(self, azrecord): + vals = [ar.nsdname for ar in azrecord.ns_records] + return {'values': [_check_endswith_dot(val) for val in vals]} + def _data_for_PTR(self, azrecord): try: ptrdname = azrecord.ptr_records[0].ptrdname @@ -366,19 +376,14 @@ class AzureProvider(BaseProvider): except: return {'value': '.'} - def _data_for_MX(self, azrecord): - return {'values': [{'preference': ar.preference, - 'exchange': ar.exchange} - for ar in azrecord.mx_records]} - def _data_for_SRV(self, azrecord): return {'values': [{'priority': ar.priority, 'weight': ar.weight, 'port': ar.port, 'target': ar.target} for ar in azrecord.srv_records]} - def _data_for_NS(self, azrecord): - vals = [ar.nsdname for ar in azrecord.ns_records] - return {'values': [_check_endswith_dot(val) for val in vals]} + def _data_for_TXT(self, azrecord): + return {'values': [reduce((lambda a, b: a + b), ar.value) + for ar in azrecord.txt_records]} def _apply_Create(self, change): '''A record from change must be created. @@ -411,8 +416,7 @@ class AzureProvider(BaseProvider): self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): - ''' - Required function of manager.py + '''Required function of manager.py to actually apply a record change. :param plan: Contains the zones and changes to be made :type plan: octodns.provider.base.Plan diff --git a/requirements.txt b/requirements.txt index 9eb8284..5d8089a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ # These are known good versions. You're free to use others and things will # likely work, but no promises are made, especilly if you go older. PyYaml==3.12 +azure-mgmt-dns==1.0.1 +azure-common==1.1.6 boto3==1.4.4 botocore==1.5.4 dnspython==1.15.0 @@ -10,12 +12,10 @@ futures==3.0.5 incf.countryutils==1.0 ipaddress==1.0.18 jmespath==0.9.0 +msrestazure==0.4.10 natsort==5.0.3 nsone==0.9.14 python-dateutil==2.6.0 requests==2.13.0 s3transfer==0.1.10 -six==1.10.0 -azure-mgmt-dns==1.0.1 -azure-common==1.1.6 -msrestazure==0.4.10 \ No newline at end of file +six==1.10.0 \ No newline at end of file diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 5ffa055..59cf551 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -16,7 +16,6 @@ from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ Zone as AzureZone from msrestazure.azure_exceptions import CloudError - from unittest import TestCase from mock import Mock, patch From 8d7b3fb101959898a576e735dea606e616d26614 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 5 Jul 2017 14:09:05 -0700 Subject: [PATCH 39/84] Remove ; escapes before sending to ns1 and when pulling from --- octodns/provider/ns1.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 65db64c..e7b8ffb 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -42,8 +42,16 @@ class Ns1Provider(BaseProvider): } _data_for_AAAA = _data_for_A - _data_for_SPF = _data_for_A - _data_for_TXT = _data_for_A + + def _data_for_SPF(self, _type, record): + values = [v.replace(';', '\;') for v in record['short_answers']] + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': values + } + + _data_for_TXT = _data_for_SPF def _data_for_CNAME(self, _type, record): return { @@ -141,8 +149,15 @@ class Ns1Provider(BaseProvider): _params_for_AAAA = _params_for_A _params_for_NS = _params_for_A - _params_for_SPF = _params_for_A - _params_for_TXT = _params_for_A + + def _params_for_SPF(self, record): + # NS1 seems to be the only provider that doesn't want things escaped in + # values so we have to strip them here and add them when going the + # other way + values = [v.replace('\\', '') for v in record.values] + return {'answers': values, 'ttl': record.ttl} + + _params_for_TXT = _params_for_SPF def _params_for_CNAME(self, record): return {'answers': [record.value], 'ttl': record.ttl} From 818c1e9cc6ba6d155d43a4f3152615d48dabe091 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 5 Jul 2017 14:28:01 -0700 Subject: [PATCH 40/84] Unit tests for ns1 escape handling and fix --- octodns/provider/ns1.py | 2 +- tests/test_octodns_provider_ns1.py | 34 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index e7b8ffb..7757812 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -154,7 +154,7 @@ class Ns1Provider(BaseProvider): # NS1 seems to be the only provider that doesn't want things escaped in # values so we have to strip them here and add them when going the # other way - values = [v.replace('\\', '') for v in record.values] + values = [v.replace('\;', ';') for v in record.values] return {'answers': values, 'ttl': record.ttl} _params_for_TXT = _params_for_SPF diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 0398459..afcdd41 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -277,3 +277,37 @@ class TestNs1Provider(TestCase): call.update(answers=[u'1.2.3.4'], ttl=32), call.delete() ]) + + def test_escaping(self): + provider = Ns1Provider('test', 'api-key') + + record = { + 'ttl': 31, + 'short_answers': ['foo; bar baz; blip'] + } + self.assertEquals(['foo\; bar baz\; blip'], + provider._data_for_SPF('SPF', record)['values']) + + record = { + 'ttl': 31, + 'short_answers': ['no', 'foo; bar baz; blip', 'yes'] + } + self.assertEquals(['no', 'foo\; bar baz\; blip', 'yes'], + provider._data_for_TXT('TXT', record)['values']) + + zone = Zone('unit.tests.', []) + record = Record.new(zone, 'spf', { + 'ttl': 34, + 'type': 'SPF', + 'value': 'foo\; bar baz\; blip' + }) + self.assertEquals(['foo; bar baz; blip'], + provider._params_for_SPF(record)['answers']) + + record = Record.new(zone, 'txt', { + 'ttl': 35, + 'type': 'TXT', + 'value': 'foo\; bar baz\; blip' + }) + self.assertEquals(['foo; bar baz; blip'], + provider._params_for_TXT(record)['answers']) From 22a05639160bf7c0417d82af78e0ac4df3d43f15 Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Mon, 10 Jul 2017 16:00:50 -0700 Subject: [PATCH 41/84] fix unsafe plan text interpolation --- octodns/provider/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 2fd4349..bcf566a 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -56,8 +56,8 @@ class Plan(object): delete_pcent = self.change_counts['Delete'] / existing_record_count if update_pcent > self.MAX_SAFE_UPDATE_PCENT: - raise UnsafePlan('Too many updates, %s is over %s percent' - '(%s/%s)', + raise UnsafePlan('Too many updates, {} is over {} percent' + '({}/{})'.format, update_pcent, self.MAX_SAFE_UPDATE_PCENT * 100, self.change_counts['Update'], From 2b58e065e8569e336aaec2cd3f759f03ed69867a Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 11 Jul 2017 07:09:20 -0700 Subject: [PATCH 42/84] fix format --- octodns/provider/base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index bcf566a..8c97edb 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -57,18 +57,18 @@ class Plan(object): if update_pcent > self.MAX_SAFE_UPDATE_PCENT: raise UnsafePlan('Too many updates, {} is over {} percent' - '({}/{})'.format, + '({}/{})'.format( update_pcent, self.MAX_SAFE_UPDATE_PCENT * 100, self.change_counts['Update'], - existing_record_count) + existing_record_count)) if delete_pcent > self.MAX_SAFE_DELETE_PCENT: - raise UnsafePlan('Too many deletes, %s is over %s percent' - '(%s/%s)', + raise UnsafePlan('Too many deletes, {} is over {} percent' + '({}/{})'.format( delete_pcent, self.MAX_SAFE_DELETE_PCENT * 100, self.change_counts['Delete'], - existing_record_count) + existing_record_count)) def __repr__(self): return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \ From 11a246da81928640387a8af36f0b3da8a93cfbe2 Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 11 Jul 2017 07:20:18 -0700 Subject: [PATCH 43/84] whitespace --- octodns/provider/base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 8c97edb..d2561ba 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -58,17 +58,17 @@ class Plan(object): if update_pcent > self.MAX_SAFE_UPDATE_PCENT: raise UnsafePlan('Too many updates, {} is over {} percent' '({}/{})'.format( - update_pcent, - self.MAX_SAFE_UPDATE_PCENT * 100, - self.change_counts['Update'], - existing_record_count)) + update_pcent, + self.MAX_SAFE_UPDATE_PCENT * 100, + self.change_counts['Update'], + existing_record_count)) if delete_pcent > self.MAX_SAFE_DELETE_PCENT: raise UnsafePlan('Too many deletes, {} is over {} percent' '({}/{})'.format( - delete_pcent, - self.MAX_SAFE_DELETE_PCENT * 100, - self.change_counts['Delete'], - existing_record_count)) + delete_pcent, + self.MAX_SAFE_DELETE_PCENT * 100, + self.change_counts['Delete'], + existing_record_count)) def __repr__(self): return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \ From 5b746845ed567169edb523a2149d385743c84c0c Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 11 Jul 2017 07:36:24 -0700 Subject: [PATCH 44/84] add tests --- tests/test_octodns_provider_base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index bd134bc..e44adc0 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -214,9 +214,11 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_UPDATE_PCENT) + 1)] - with self.assertRaises(UnsafePlan): + with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes).raise_if_unsafe() + self.assertTrue('Too many updates' in ctx.exception.message) + def test_safe_updates_min_existing_pcent(self): # MAX_SAFE_UPDATE_PCENT is safe when more # than MIN_EXISTING_RECORDS exist @@ -260,9 +262,11 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_DELETE_PCENT) + 1)] - with self.assertRaises(UnsafePlan): + with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes).raise_if_unsafe() + self.assertTrue('Too many deletes' in ctx.exception.message) + def test_safe_deletes_min_existing_pcent(self): # MAX_SAFE_DELETE_PCENT is safe when more # than MIN_EXISTING_RECORDS exist From f0258d4b2a1bfcc8b3b1b8236f15ccfcfa4a3c61 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 21 Jul 2017 08:53:21 -0700 Subject: [PATCH 45/84] Release v0.8.5 --- CHANGELOG.md | 15 +++++++++++++-- octodns/__init__.py | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1cb1204..d58d4de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ +## v0.8.5 - 2017-07-21 - Azure, NS1 escaping, & large zones -## v0.8.4 - 2017-03-14 - It's been too long +Relatively small delta this go around. No major themes or anything, just steady +progress. + +* AzureProvider added thanks to work by + [Heesu Hwang](https://github.com/h-hwang). +* Fixed some escaping issues with NS1 TXT and SPF records that were tracked down + with the help of [Blake Stoddard](https://github.com/blakestoddard). +* Some tweaks were made to Zone.records to vastly improve handling of zones with + very large numbers of records, no more O(N^2). + +## v0.8.4 - 2017-06-28 - It's been too long Lots of updates based on our internal use, needs, and feedback & suggestions from our OSS users. There's too much to list out since the previous release was @@ -28,7 +39,7 @@ better in the future :fingers_crossed: implementation. Sorting can be disabled in the YamlProvider with `enforce_order: False`. * Semi-colon/escaping fixes and improvements. -* Meta record support, `TXT octodns-meta.`. For now just +* Meta record support, `TXT octodns-meta.`. For now just `provider=`. Optionally turned on with `include_meta` manager config val. * Validations check for CNAMEs co-existing with other records and error out if diff --git a/octodns/__init__.py b/octodns/__init__.py index 94ef299..601734c 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.8.4' +__VERSION__ = '0.8.5' From 7f8a01a81da107fb0babffeaed49ba67372e3bfc Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Tue, 25 Jul 2017 09:15:30 -0700 Subject: [PATCH 46/84] Improved/actionable keys out of order error message --- octodns/yaml.py | 10 +++++++--- tests/test_octodns_yaml.py | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/octodns/yaml.py b/octodns/yaml.py index d4ab541..98bafdb 100644 --- a/octodns/yaml.py +++ b/octodns/yaml.py @@ -21,9 +21,13 @@ class SortEnforcingLoader(SafeLoader): self.flatten_mapping(node) ret = self.construct_pairs(node) keys = [d[0] for d in ret] - if keys != sorted(keys, key=_natsort_key): - raise ConstructorError(None, None, "keys out of order: {}" - .format(', '.join(keys)), node.start_mark) + keys_sorted = sorted(keys, key=_natsort_key) + for key in keys: + expected = keys_sorted.pop(0) + if key != expected: + raise ConstructorError(None, None, 'keys out of order: ' + 'expected {} got {} at {}' + .format(expected, key, node.start_mark)) return dict(ret) diff --git a/tests/test_octodns_yaml.py b/tests/test_octodns_yaml.py index 0f454b3..effe231 100644 --- a/tests/test_octodns_yaml.py +++ b/tests/test_octodns_yaml.py @@ -48,8 +48,8 @@ class TestYaml(TestCase): '*.11.2': 'd' '*.10.1': 'c' ''') - self.assertEquals('keys out of order: *.2.2, *.1.2, *.11.2, *.10.1', - ctx.exception.problem) + self.assertTrue('keys out of order: expected *.1.2 got *.2.2 at' in + ctx.exception.problem) buf = StringIO() safe_dump({ From 268620c9398713252298e0685fa149dfbb635538 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 12 Aug 2017 12:54:53 -0700 Subject: [PATCH 47/84] Add support for increasing Route53 retries --- octodns/provider/route53.py | 13 +++++++++++-- requirements.txt | 14 +++++++------- tests/test_octodns_provider_route53.py | 8 ++++++++ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 490d630..6f9adc2 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from boto3 import client +from botocore.config import Config from collections import defaultdict from incf.countryutils.transformations import cca_to_ctca2 from uuid import uuid4 @@ -229,14 +230,22 @@ class Route53Provider(BaseProvider): HEALTH_CHECK_VERSION = '0000' def __init__(self, id, access_key_id, secret_access_key, max_changes=1000, - *args, **kwargs): + client_max_attempts=None, *args, **kwargs): self.max_changes = max_changes self.log = logging.getLogger('Route53Provider[{}]'.format(id)) self.log.debug('__init__: id=%s, access_key_id=%s, ' 'secret_access_key=***', id, access_key_id) super(Route53Provider, self).__init__(id, *args, **kwargs) + + config = None + if client_max_attempts is not None: + self.log.info('__init__: setting max_attempts to %d', + client_max_attempts) + config = Config(retries={'max_attempts': client_max_attempts}) + self._conn = client('route53', aws_access_key_id=access_key_id, - aws_secret_access_key=secret_access_key) + aws_secret_access_key=secret_access_key, + config=config) self._r53_zones = None self._r53_rrsets = {} diff --git a/requirements.txt b/requirements.txt index 5d8089a..2aec6d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,19 +3,19 @@ PyYaml==3.12 azure-mgmt-dns==1.0.1 azure-common==1.1.6 -boto3==1.4.4 -botocore==1.5.4 +boto3==1.4.6 +botocore==1.6.0 dnspython==1.15.0 -docutils==0.13.1 +docutils==0.14 dyn==1.7.10 -futures==3.0.5 +futures==3.1.1 incf.countryutils==1.0 ipaddress==1.0.18 -jmespath==0.9.0 +jmespath==0.9.3 msrestazure==0.4.10 natsort==5.0.3 nsone==0.9.14 -python-dateutil==2.6.0 +python-dateutil==2.6.1 requests==2.13.0 s3transfer==0.1.10 -six==1.10.0 \ No newline at end of file +six==1.10.0 diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index be624ff..97dae4f 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -1232,6 +1232,14 @@ class TestRoute53Provider(TestCase): 'Type': 'TXT', })) + def test_client_max_attempts(self): + provider = Route53Provider('test', 'abc', '123', + client_max_attempts=42) + # NOTE: this will break if boto ever changes the impl details... + self.assertEquals(43, provider._conn.meta.events + ._unique_id_handlers['retry-config-route53'] + ['handler']._checker.__dict__['_max_attempts']) + class TestRoute53Records(TestCase): From 75ca21a6cd6cba22ad773a73ce9c028f8e07890a Mon Sep 17 00:00:00 2001 From: Patrick O'Brien Date: Tue, 15 Aug 2017 16:09:05 -0700 Subject: [PATCH 48/84] Allow scheme to be specified for powerdns This allows a scheme to be set for the PowerDNS API. It defaults to http to retain backwards compatibility. --- octodns/provider/powerdns.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index c6d11b0..62b6fd8 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -18,11 +18,13 @@ class PowerDnsBaseProvider(BaseProvider): 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 - def __init__(self, id, host, api_key, port=8081, *args, **kwargs): + def __init__(self, id, host, api_key, port=8081, scheme="http", *args, + **kwargs): super(PowerDnsBaseProvider, self).__init__(id, *args, **kwargs) self.host = host self.port = port + self.scheme = scheme sess = Session() sess.headers.update({'X-API-Key': api_key}) @@ -31,8 +33,8 @@ class PowerDnsBaseProvider(BaseProvider): def _request(self, method, path, data=None): self.log.debug('_request: method=%s, path=%s', method, path) - url = 'http://{}:{}/api/v1/servers/localhost/{}' \ - .format(self.host, self.port, path) + url = '{}://{}:{}/api/v1/servers/localhost/{}' \ + .format(self.scheme, self.host, self.port, path) resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() From a2c9950d28ad40d548a0ad5f4ef27c8cdd9af97d Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 21 Aug 2017 10:28:43 -0700 Subject: [PATCH 49/84] Fixed inconsistency bug with adding TXT records with Azure. --- octodns/provider/azuredns.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4433b1e..ee3a6ed 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -90,7 +90,6 @@ class _AzureRecord(object): _params_for_AAAA = _params _params_for_NS = _params _params_for_PTR = _params - _params_for_TXT = _params def _params_for_CNAME(self, data, key_name, azure_class): return {key_name: azure_class(data['value'])} @@ -121,6 +120,12 @@ class _AzureRecord(object): data['value']['target'])) return {key_name: params} + def _params_for_TXT(self, data, key_name, azure_class): + if 'values' in data: + return {key_name: [azure_class([v]) for v in data['values']]} + else: # API for TxtRecord has list of str, even for singleton + return {key_name: [azure_class([data['value']])]} + def _equals(self, b): '''Checks whether two records are equal by comparing all fields. :param b: Another _AzureRecord object From 9623f4e7833dc546527e8031597acd08b7052a1d Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 21 Aug 2017 11:02:28 -0700 Subject: [PATCH 50/84] updated testfile to include test cases for new TXT data parsing --- tests/test_octodns_provider_azuredns.py | 33 +++++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 59cf551..598fe48 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -90,6 +90,14 @@ octo_records.append(Record.new(zone, '_srv2._tcp', { 'port': 1, 'target': 'srvfoo.unit.tests.', }]})) +octo_records.append(Record.new(zone, 'txt1', { + 'ttl': 8, + 'type': 'TXT', + 'value': 'txt singleton test'})) +octo_records.append(Record.new(zone, 'txt2', { + 'ttl': 9, + 'type': 'TXT', + 'values': ['txt multiple test', 'txt multiple test 2']})) azure_records = [] _base0 = _AzureRecord('TestAzure', octo_records[0]) @@ -183,6 +191,23 @@ _base10.params['ttl'] = 7 _base10.params['srv_records'] = [SrvRecord(12, 17, 1, 'srvfoo.unit.tests.')] azure_records.append(_base10) +_base11 = _AzureRecord('TestAzure', octo_records[11]) +_base11.zone_name = 'unit.tests' +_base11.relative_record_set_name = 'txt1' +_base11.record_type = 'TXT' +_base11.params['ttl'] = 8 +_base11.params['txt_records'] = [TxtRecord(['txt singleton test'])] +azure_records.append(_base11) + +_base12 = _AzureRecord('TestAzure', octo_records[12]) +_base12.zone_name = 'unit.tests' +_base12.relative_record_set_name = 'txt2' +_base12.record_type = 'TXT' +_base12.params['ttl'] = 9 +_base12.params['txt_records'] = [TxtRecord(['txt multiple test']), + TxtRecord(['txt multiple test 2'])] +azure_records.append(_base12) + class Test_AzureRecord(TestCase): def test_azure_record(self): @@ -190,8 +215,6 @@ class Test_AzureRecord(TestCase): for i in range(len(azure_records)): octo = _AzureRecord('TestAzure', octo_records[i]) assert(azure_records[i]._equals(octo)) - string = str(azure_records[i]) - assert(('Ttl: ' in string)) class Test_ParseAzureType(TestCase): @@ -315,8 +338,8 @@ class TestAzureDnsProvider(TestCase): changes.append(Create(i)) deletes.append(Delete(i)) - self.assertEquals(11, provider.apply(Plan(None, zone, changes))) - self.assertEquals(11, provider.apply(Plan(zone, zone, deletes))) + self.assertEquals(13, provider.apply(Plan(None, zone, changes))) + self.assertEquals(13, provider.apply(Plan(zone, zone, deletes))) def test_create_zone(self): provider = self._get_provider() @@ -331,7 +354,7 @@ class TestAzureDnsProvider(TestCase): _get = provider._dns_client.zones.get _get.side_effect = CloudError(Mock(status=404), err_msg) - self.assertEquals(11, provider.apply(Plan(None, desired, changes))) + self.assertEquals(13, provider.apply(Plan(None, desired, changes))) def test_check_zone_no_create(self): provider = self._get_provider() From a46ee23cc5742c368037d98b8b9b88c65471f139 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 21 Aug 2017 11:59:18 -0700 Subject: [PATCH 51/84] Slight refactor to make parsing value vs values separate from return --- octodns/provider/azuredns.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index ee3a6ed..1757274 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -81,10 +81,11 @@ class _AzureRecord(object): self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): - if 'values' in data: - return {key_name: [azure_class(v) for v in data['values']]} - else: # Else there is a singular data point keyed by 'value'. - return {key_name: [azure_class(data['value'])]} + try: + values = data['values'] + except KeyError: + values = [data['value']] + return {key_name: [azure_class(v) for v in values]} _params_for_A = _params _params_for_AAAA = _params @@ -121,10 +122,11 @@ class _AzureRecord(object): return {key_name: params} def _params_for_TXT(self, data, key_name, azure_class): - if 'values' in data: - return {key_name: [azure_class([v]) for v in data['values']]} - else: # API for TxtRecord has list of str, even for singleton - return {key_name: [azure_class([data['value']])]} + try: # API for TxtRecord has list of str, even for singleton + values = data['values'] + except KeyError: + values = [data['value']] + return {key_name: [azure_class([v]) for v in values]} def _equals(self, b): '''Checks whether two records are equal by comparing all fields. From 4cae1e2bdb71b9e4d4298c727a51932ec3ef8b67 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 08:18:17 -0700 Subject: [PATCH 52/84] Add CAA Record class and tests --- octodns/record.py | 75 +++++++++++++++---- tests/test_octodns_record.py | 138 ++++++++++++++++++++++++++++++++++- 2 files changed, 195 insertions(+), 18 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index 6ee9dff..cc9949f 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -81,29 +81,16 @@ class Record(object): 'A': ARecord, 'AAAA': AaaaRecord, 'ALIAS': AliasRecord, - # cert + 'CAA': CaaRecord, 'CNAME': CnameRecord, - # dhcid - # dname - # dnskey - # ds - # ipseckey - # key - # kx - # loc 'MX': MxRecord, 'NAPTR': NaptrRecord, 'NS': NsRecord, - # nsap 'PTR': PtrRecord, - # px - # rp - # soa - would it even make sense? 'SPF': SpfRecord, 'SRV': SrvRecord, 'SSHFP': SshfpRecord, 'TXT': TxtRecord, - # url }[_type] except KeyError: raise Exception('Unknown record type: "{}"'.format(_type)) @@ -398,6 +385,66 @@ class AliasRecord(_ValueMixin, Record): return value +class CaaValue(object): + # https://tools.ietf.org/html/rfc6844#page-5 + + @classmethod + def _validate_value(cls, value): + reasons = [] + try: + flags = int(value.get('flags', 0)) + if flags not in (0, 1): + reasons.append('invalid flags "{}"'.format(flags)) + except ValueError: + reasons.append('invalid flags "{}"'.format(value['flags'])) + + try: + tag = value['tag'] + if tag not in ('issue', 'issuewild', 'iodef'): + reasons.append('invalid tag "{}"'.format(tag)) + except KeyError: + reasons.append('missing tag') + + if 'value' not in value: + reasons.append('missing value') + + return reasons + + def __init__(self, value): + self.flags = int(value.get('flags', 0)) + self.tag = value['tag'] + self.value = value['value'] + + @property + def data(self): + return { + 'flags': self.flags, + 'tag': self.tag, + 'value': self.value, + } + + def __cmp__(self, other): + if self.flags == other.flags: + if self.tag == other.tag: + return cmp(self.value, other.value) + return cmp(self.tag, other.tag) + return cmp(self.flags, other.flags) + + def __repr__(self): + return "'{} {} {}'".format(self.flags, self.tag, self.value) + + +class CaaRecord(_ValuesMixin, Record): + _type = 'CAA' + + @classmethod + def _validate_value(cls, value): + return CaaValue._validate_value(value) + + def _process_values(self, values): + return [CaaValue(v) for v in values] + + class CnameRecord(_ValueMixin, Record): _type = 'CNAME' diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 1d64081..10e3869 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -7,10 +7,10 @@ from __future__ import absolute_import, division, print_function, \ from unittest import TestCase -from octodns.record import ARecord, AaaaRecord, AliasRecord, CnameRecord, \ - Create, Delete, GeoValue, MxRecord, NaptrRecord, NaptrValue, NsRecord, \ - Record, SshfpRecord, SpfRecord, SrvRecord, TxtRecord, Update, \ - ValidationError +from octodns.record import ARecord, AaaaRecord, AliasRecord, CaaRecord, \ + CnameRecord, Create, Delete, GeoValue, MxRecord, NaptrRecord, \ + NaptrValue, NsRecord, Record, SshfpRecord, SpfRecord, SrvRecord, \ + TxtRecord, Update, ValidationError from octodns.zone import Zone from helpers import GeoProvider, SimpleProvider @@ -206,6 +206,66 @@ class TestRecord(TestCase): # __repr__ doesn't blow up a.__repr__() + def test_caa(self): + a_values = [{ + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.example.net', + }, { + 'flags': 1, + 'tag': 'iodef', + 'value': 'mailto:security@example.com', + }] + a_data = {'ttl': 30, 'values': a_values} + a = CaaRecord(self.zone, 'a', a_data) + self.assertEquals('a', a.name) + self.assertEquals('a.unit.tests.', a.fqdn) + self.assertEquals(30, a.ttl) + self.assertEquals(a_values[0]['flags'], a.values[0].flags) + self.assertEquals(a_values[0]['tag'], a.values[0].tag) + self.assertEquals(a_values[0]['value'], a.values[0].value) + self.assertEquals(a_values[1]['flags'], a.values[1].flags) + self.assertEquals(a_values[1]['tag'], a.values[1].tag) + self.assertEquals(a_values[1]['value'], a.values[1].value) + self.assertEquals(a_data, a.data) + + b_value = { + 'tag': 'iodef', + 'value': 'http://iodef.example.com/', + } + b_data = {'ttl': 30, 'value': b_value} + b = CaaRecord(self.zone, 'b', b_data) + self.assertEquals(0, b.values[0].flags) + self.assertEquals(b_value['tag'], b.values[0].tag) + self.assertEquals(b_value['value'], b.values[0].value) + b_data['value']['flags'] = 0 + self.assertEquals(b_data, b.data) + + target = SimpleProvider() + # No changes with self + self.assertFalse(a.changes(a, target)) + # Diff in flags causes change + other = CaaRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) + other.values[0].flags = 1 + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in tag causes change + other.values[0].flags = a.values[0].flags + other.values[0].tag = 'foo' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + # Diff in value causes change + other.values[0].tag = a.values[0].tag + other.values[0].value = 'bar' + change = a.changes(other, target) + self.assertEqual(change.existing, a) + self.assertEqual(change.new, other) + + # __repr__ doesn't blow up + a.__repr__() + def test_cname(self): self.assertSingleValue(CnameRecord, 'target.foo.com.', 'other.foo.com.') @@ -861,6 +921,76 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['missing trailing .'], ctx.exception.reasons) + def test_CAA(self): + # doesn't blow up + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': 1, + 'tag': 'iodef', + 'value': 'http://foo.bar.com/' + } + }) + + # invalid flags + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': 42, + 'tag': 'iodef', + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['invalid flags "42"'], ctx.exception.reasons) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': 'nope', + 'tag': 'iodef', + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['invalid flags "nope"'], ctx.exception.reasons) + + # missing tag + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['missing tag'], ctx.exception.reasons) + + # invalid tag + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'tag': 'xyz', + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['invalid tag "xyz"'], ctx.exception.reasons) + + # missing value + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'tag': 'iodef', + } + }) + self.assertEquals(['missing value'], ctx.exception.reasons) + def test_CNAME(self): # doesn't blow up Record.new(self.zone, 'www', { From 1e68cd6ae98a8d6891010174bd73992e68fbae58 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 09:03:59 -0700 Subject: [PATCH 53/84] Add CAA support to Dyn, PowerDNS, and Route53 --- octodns/provider/dyn.py | 16 ++++++++++++++++ octodns/provider/powerdns.py | 25 +++++++++++++++++++++++-- octodns/provider/route53.py | 23 +++++++++++++++++++++-- octodns/record.py | 11 +++-------- requirements.txt | 4 ++-- tests/test_octodns_record.py | 18 +++--------------- 6 files changed, 68 insertions(+), 29 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index e21b93e..3b7b9ea 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -111,6 +111,7 @@ class DynProvider(BaseProvider): 'a_records': 'A', 'aaaa_records': 'AAAA', 'alias_records': 'ALIAS', + 'caa_records': 'CAA', 'cname_records': 'CNAME', 'mx_records': 'MX', 'naptr_records': 'NAPTR', @@ -194,6 +195,14 @@ class DynProvider(BaseProvider): 'value': record.alias } + def _data_for_CAA(self, _type, records): + return { + 'type': _type, + 'ttl': records[0].ttl, + 'values': [{'flags': r.flags, 'tag': r.tag, 'value': r.value} + for r in records], + } + def _data_for_CNAME(self, _type, records): record = records[0] return { @@ -382,6 +391,13 @@ class DynProvider(BaseProvider): _kwargs_for_AAAA = _kwargs_for_A + def _kwargs_for_CAA(self, record): + return [{ + 'flags': v.flags, + 'tag': v.tag, + 'value': v.value, + } for v in record.values] + def _kwargs_for_CNAME(self, record): return [{ 'cname': record.value, diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 62b6fd8..55ca0b1 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -14,8 +14,8 @@ from .base import BaseProvider class PowerDnsBaseProvider(BaseProvider): SUPPORTS_GEO = False - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', - 'SPF', 'SSHFP', 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', + 'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 def __init__(self, id, host, api_key, port=8081, scheme="http", *args, @@ -61,6 +61,21 @@ class PowerDnsBaseProvider(BaseProvider): _data_for_AAAA = _data_for_multiple _data_for_NS = _data_for_multiple + def _data_for_CAA(self, rrset): + values = [] + for record in rrset['records']: + flags, tag, value = record['content'].split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value, + }) + return { + 'type': rrset['type'], + 'values': values, + 'ttl': rrset['ttl'] + } + def _data_for_single(self, rrset): return { 'type': rrset['type'], @@ -194,6 +209,12 @@ class PowerDnsBaseProvider(BaseProvider): _records_for_AAAA = _records_for_multiple _records_for_NS = _records_for_multiple + def _records_for_CAA(self, record): + return [{ + 'content': '{} {} "{}"'.format(v.flags, v.tag, v.value), + 'disabled': False + } for v in record.values] + def _records_for_single(self, record): return [{'content': record.value, 'disabled': False}] diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 6f9adc2..0600511 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -90,6 +90,10 @@ class _Route53Record(object): _values_for_AAAA = _values_for_values _values_for_NS = _values_for_values + def _values_for_CAA(self, record): + return ['{} {} "{}"'.format(v.flags, v.tag, v.value) + for v in record.values] + def _values_for_value(self, record): return [record.value] @@ -222,8 +226,8 @@ class Route53Provider(BaseProvider): In general the account used will need full permissions on Route53. ''' SUPPORTS_GEO = True - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', - 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', + 'SPF', 'SRV', 'TXT')) # This should be bumped when there are underlying changes made to the # health check config. @@ -319,6 +323,21 @@ class Route53Provider(BaseProvider): _data_for_A = _data_for_geo _data_for_AAAA = _data_for_geo + def _data_for_CAA(self, rrset): + values = [] + for rr in rrset['ResourceRecords']: + flags, tag, value = rr['Value'].split(' ') + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value[1:-1], + }) + return { + 'type': rrset['Type'], + 'values': values, + 'ttl': int(rrset['TTL']) + } + def _data_for_single(self, rrset): return { 'type': rrset['Type'], diff --git a/octodns/record.py b/octodns/record.py index cc9949f..aa41606 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -393,18 +393,13 @@ class CaaValue(object): reasons = [] try: flags = int(value.get('flags', 0)) - if flags not in (0, 1): + if flags not in (0, 128): reasons.append('invalid flags "{}"'.format(flags)) except ValueError: reasons.append('invalid flags "{}"'.format(value['flags'])) - try: - tag = value['tag'] - if tag not in ('issue', 'issuewild', 'iodef'): - reasons.append('invalid tag "{}"'.format(tag)) - except KeyError: + if 'tag' not in value: reasons.append('missing tag') - if 'value' not in value: reasons.append('missing value') @@ -431,7 +426,7 @@ class CaaValue(object): return cmp(self.flags, other.flags) def __repr__(self): - return "'{} {} {}'".format(self.flags, self.tag, self.value) + return '{} {} "{}"'.format(self.flags, self.tag, self.value) class CaaRecord(_ValuesMixin, Record): diff --git a/requirements.txt b/requirements.txt index 2aec6d0..d2be70f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,10 +4,10 @@ PyYaml==3.12 azure-mgmt-dns==1.0.1 azure-common==1.1.6 boto3==1.4.6 -botocore==1.6.0 +botocore==1.6.8 dnspython==1.15.0 docutils==0.14 -dyn==1.7.10 +dyn==1.8.0 futures==3.1.1 incf.countryutils==1.0 ipaddress==1.0.18 diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 10e3869..215abe1 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -212,7 +212,7 @@ class TestRecord(TestCase): 'tag': 'issue', 'value': 'ca.example.net', }, { - 'flags': 1, + 'flags': 128, 'tag': 'iodef', 'value': 'mailto:security@example.com', }] @@ -246,7 +246,7 @@ class TestRecord(TestCase): self.assertFalse(a.changes(a, target)) # Diff in flags causes change other = CaaRecord(self.zone, 'a', {'ttl': 30, 'values': a_values}) - other.values[0].flags = 1 + other.values[0].flags = 128 change = a.changes(other, target) self.assertEqual(change.existing, a) self.assertEqual(change.new, other) @@ -927,7 +927,7 @@ class TestRecordValidation(TestCase): 'type': 'CAA', 'ttl': 600, 'value': { - 'flags': 1, + 'flags': 128, 'tag': 'iodef', 'value': 'http://foo.bar.com/' } @@ -968,18 +968,6 @@ class TestRecordValidation(TestCase): }) self.assertEquals(['missing tag'], ctx.exception.reasons) - # invalid tag - with self.assertRaises(ValidationError) as ctx: - Record.new(self.zone, '', { - 'type': 'CAA', - 'ttl': 600, - 'value': { - 'tag': 'xyz', - 'value': 'http://foo.bar.com/', - } - }) - self.assertEquals(['invalid tag "xyz"'], ctx.exception.reasons) - # missing value with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { From 22591ae84b80812b0f01bc3c7c7fc12b18310869 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 09:38:21 -0700 Subject: [PATCH 54/84] Add CAA support for NS1 --- octodns/provider/ns1.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 7757812..f7cbef1 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -23,8 +23,8 @@ class Ns1Provider(BaseProvider): api_key: env/NS1_API_KEY ''' SUPPORTS_GEO = False - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', - 'SPF', 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', + 'PTR', 'SPF', 'SRV', 'TXT')) ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' @@ -53,6 +53,21 @@ class Ns1Provider(BaseProvider): _data_for_TXT = _data_for_SPF + def _data_for_CAA(self, _type, record): + values = [] + for answer in record['short_answers']: + flags, tag, value = answer.split(' ', 2) + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value, + }) + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': values, + } + def _data_for_CNAME(self, _type, record): return { 'ttl': record['ttl'], @@ -159,6 +174,10 @@ class Ns1Provider(BaseProvider): _params_for_TXT = _params_for_SPF + def _params_for_CAA(self, record): + values = [(v.flags, v.tag, v.value) for v in record.values] + return {'answers': values, 'ttl': record.ttl} + def _params_for_CNAME(self, record): return {'answers': [record.value], 'ttl': record.ttl} From c24c793bcbe59acdac97b819e649c8dceca103fe Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 15:28:09 -0700 Subject: [PATCH 55/84] CAA unit tests for provider support --- octodns/provider/powerdns.py | 2 +- tests/config/unit.tests.yaml | 5 +++++ tests/fixtures/powerdns-full-data.json | 12 ++++++++++ tests/test_octodns_provider_dyn.py | 22 +++++++++++++++++-- tests/test_octodns_provider_ns1.py | 14 ++++++++++++ tests/test_octodns_provider_powerdns.py | 6 ++--- tests/test_octodns_provider_route53.py | 29 ++++++++++++++++++------- tests/test_octodns_provider_yaml.py | 2 +- 8 files changed, 77 insertions(+), 15 deletions(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 55ca0b1..20cfe8b 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -68,7 +68,7 @@ class PowerDnsBaseProvider(BaseProvider): values.append({ 'flags': flags, 'tag': tag, - 'value': value, + 'value': value[1:-1], }) return { 'type': rrset['type'], diff --git a/tests/config/unit.tests.yaml b/tests/config/unit.tests.yaml index 8be1614..5241406 100644 --- a/tests/config/unit.tests.yaml +++ b/tests/config/unit.tests.yaml @@ -31,6 +31,11 @@ values: - 6.2.3.4. - 7.2.3.4. + - type: CAA + values: + - flags: 0 + tag: issue + value: ca.unit.tests _srv._tcp: ttl: 600 type: SRV diff --git a/tests/fixtures/powerdns-full-data.json b/tests/fixtures/powerdns-full-data.json index 72ce016..b8f8bf3 100644 --- a/tests/fixtures/powerdns-full-data.json +++ b/tests/fixtures/powerdns-full-data.json @@ -230,6 +230,18 @@ ], "ttl": 300, "type": "A" + }, + { + "comments": [], + "name": "unit.tests.", + "records": [ + { + "content": "0 issue \"ca.unit.tests\"", + "disabled": false + } + ], + "ttl": 3600, + "type": "CAA" } ], "serial": 2017012803, diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index bebd3e3..9be253d 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -109,6 +109,14 @@ class TestDynProvider(TestCase): 'weight': 22, 'port': 20, 'target': 'foo-2.unit.tests.' + }]}), + ('', { + 'type': 'CAA', + 'ttl': 308, + 'values': [{ + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests' }]})): expected.add_record(Record.new(expected, name, data)) @@ -321,6 +329,16 @@ class TestDynProvider(TestCase): 'ttl': 307, 'zone': 'unit.tests', }], + 'caa_records': [{ + 'fqdn': 'unit.tests', + 'rdata': {'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests'}, + 'record_id': 12, + 'record_type': 'cAA', + 'ttl': 308, + 'zone': 'unit.tests', + }], }} ] got = Zone('unit.tests.', []) @@ -414,10 +432,10 @@ class TestDynProvider(TestCase): update_mock.assert_called() add_mock.assert_called() # Once for each dyn record (8 Records, 2 of which have dual values) - self.assertEquals(14, len(add_mock.call_args_list)) + self.assertEquals(15, len(add_mock.call_args_list)) execute_mock.assert_has_calls([call('/Zone/unit.tests/', 'GET', {}), call('/Zone/unit.tests/', 'GET', {})]) - self.assertEquals(9, len(plan.changes)) + self.assertEquals(10, len(plan.changes)) execute_mock.reset_mock() diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 4df56b3..cde23b0 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -96,6 +96,15 @@ class TestNs1Provider(TestCase): 'type': 'NS', 'values': ['ns3.unit.tests.', 'ns4.unit.tests.'], })) + expected.add(Record.new(zone, '', { + 'ttl': 40, + 'type': 'CAA', + 'value': { + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests', + }, + })) nsone_records = [{ 'type': 'A', @@ -141,6 +150,11 @@ class TestNs1Provider(TestCase): 'ttl': 39, 'short_answers': ['ns3.unit.tests.', 'ns4.unit.tests.'], 'domain': 'sub.unit.tests.', + }, { + 'type': 'CAA', + 'ttl': 40, + 'short_answers': ['0 issue ca.unit.tests'], + 'domain': 'unit.tests.', }] @patch('nsone.NSONE.loadZone') diff --git a/tests/test_octodns_provider_powerdns.py b/tests/test_octodns_provider_powerdns.py index 5fcd80a..b6e02ff 100644 --- a/tests/test_octodns_provider_powerdns.py +++ b/tests/test_octodns_provider_powerdns.py @@ -79,7 +79,7 @@ class TestPowerDnsProvider(TestCase): source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) expected_n = len(expected.records) - 1 - self.assertEquals(14, expected_n) + self.assertEquals(15, expected_n) # No diffs == no changes with requests_mock() as mock: @@ -87,7 +87,7 @@ class TestPowerDnsProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -167,7 +167,7 @@ class TestPowerDnsProvider(TestCase): expected = Zone('unit.tests.', []) source = YamlProvider('test', join(dirname(__file__), 'config')) source.populate(expected) - self.assertEquals(15, len(expected.records)) + self.assertEquals(16, len(expected.records)) # A small change to a single record with requests_mock() as mock: diff --git a/tests/test_octodns_provider_route53.py b/tests/test_octodns_provider_route53.py index 97dae4f..1cd4548 100644 --- a/tests/test_octodns_provider_route53.py +++ b/tests/test_octodns_provider_route53.py @@ -77,6 +77,12 @@ class TestRoute53Provider(TestCase): {'ttl': 67, 'type': 'NS', 'values': ['8.2.3.4.', '9.2.3.4.']}), ('sub', {'ttl': 68, 'type': 'NS', 'values': ['5.2.3.4.', '6.2.3.4.']}), + ('', + {'ttl': 69, 'type': 'CAA', 'value': { + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests' + }}), ): record = Record.new(expected, name, data) expected.add_record(record) @@ -300,6 +306,13 @@ class TestRoute53Provider(TestCase): 'Value': 'ns1.unit.tests.', }], 'TTL': 69, + }, { + 'Name': 'unit.tests.', + 'Type': 'CAA', + 'ResourceRecords': [{ + 'Value': '0 issue "ca.unit.tests"', + }], + 'TTL': 69, }], 'IsTruncated': False, 'MaxItems': '100', @@ -347,7 +360,7 @@ class TestRoute53Provider(TestCase): {'HostedZoneId': 'z42'}) plan = provider.plan(self.expected) - self.assertEquals(8, len(plan.changes)) + self.assertEquals(9, len(plan.changes)) for change in plan.changes: self.assertIsInstance(change, Create) stubber.assert_no_pending_responses() @@ -366,7 +379,7 @@ class TestRoute53Provider(TestCase): 'SubmittedAt': '2017-01-29T01:02:03Z', }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY}) - self.assertEquals(8, provider.apply(plan)) + self.assertEquals(9, provider.apply(plan)) stubber.assert_no_pending_responses() # Delete by monkey patching in a populate that includes an extra record @@ -579,7 +592,7 @@ class TestRoute53Provider(TestCase): {}) plan = provider.plan(self.expected) - self.assertEquals(8, len(plan.changes)) + self.assertEquals(9, len(plan.changes)) for change in plan.changes: self.assertIsInstance(change, Create) stubber.assert_no_pending_responses() @@ -626,7 +639,7 @@ class TestRoute53Provider(TestCase): 'SubmittedAt': '2017-01-29T01:02:03Z', }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY}) - self.assertEquals(8, provider.apply(plan)) + self.assertEquals(9, provider.apply(plan)) stubber.assert_no_pending_responses() def test_health_checks_pagination(self): @@ -1174,16 +1187,16 @@ class TestRoute53Provider(TestCase): @patch('octodns.provider.route53.Route53Provider._really_apply') def test_apply_1(self, really_apply_mock): - # 17 RRs with max of 18 should only get applied in one call - provider, plan = self._get_test_plan(18) + # 18 RRs with max of 19 should only get applied in one call + provider, plan = self._get_test_plan(19) provider.apply(plan) really_apply_mock.assert_called_once() @patch('octodns.provider.route53.Route53Provider._really_apply') def test_apply_2(self, really_apply_mock): - # 17 RRs with max of 17 should only get applied in two calls - provider, plan = self._get_test_plan(17) + # 18 RRs with max of 17 should only get applied in two calls + provider, plan = self._get_test_plan(18) provider.apply(plan) self.assertEquals(2, really_apply_mock.call_count) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index 9438f01..36cd8d6 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -30,7 +30,7 @@ class TestYamlProvider(TestCase): # without it we see everything source.populate(zone) - self.assertEquals(15, len(zone.records)) + self.assertEquals(16, len(zone.records)) # Assumption here is that a clean round-trip means that everything # worked as expected, data that went in came back out and could be From f5ad26e1f989501fb415ca6ddbbadc1e12d13836 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 15:31:57 -0700 Subject: [PATCH 56/84] Fixes for dnsimple CAA support --- tests/test_octodns_provider_dnsimple.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_octodns_provider_dnsimple.py b/tests/test_octodns_provider_dnsimple.py index aed1e8b..950d460 100644 --- a/tests/test_octodns_provider_dnsimple.py +++ b/tests/test_octodns_provider_dnsimple.py @@ -78,14 +78,14 @@ class TestDnsimpleProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(14, len(zone.records)) + self.assertEquals(15, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) # 2nd populate makes no network calls/all from cache again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(14, len(again.records)) + self.assertEquals(15, len(again.records)) # bust the cache del provider._zone_records[zone.name] @@ -147,7 +147,7 @@ class TestDnsimpleProvider(TestCase): }), ]) # expected number of total calls - self.assertEquals(26, provider._client._request.call_count) + self.assertEquals(27, provider._client._request.call_count) provider._client._request.reset_mock() From e43da949a3663d04f40d66633f29c85507cafaa7 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sat, 26 Aug 2017 15:39:54 -0700 Subject: [PATCH 57/84] Add CAA for CF, DNSimple, and README --- README.md | 6 ++-- octodns/provider/cloudflare.py | 24 ++++++++++++++- octodns/provider/dnsimple.py | 29 +++++++++++++++++-- .../cloudflare-dns_records-page-2.json | 23 +++++++++++++-- tests/fixtures/dnsimple-page-2.json | 18 +++++++++++- tests/test_octodns_provider_cloudflare.py | 12 ++++---- 6 files changed, 97 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 05cf979..1f103f1 100644 --- a/README.md +++ b/README.md @@ -150,12 +150,12 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Record Support | GeoDNS Support | Notes | |--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | | -| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CNAME, MX, NS, SPF, TXT | No | | -| [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | | +| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted | +| [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | | [Ns1Provider](/octodns/provider/ns1.py) | All | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | | -| [Route53](/octodns/provider/route53.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | | +| [Route53](/octodns/provider/route53.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | | | [TinyDNSSource](/octodns/source/tinydns.py) | A, CNAME, MX, NS, PTR | No | read-only | | [YamlProvider](/octodns/provider/yaml.py) | All | Yes | config | diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index 2ee8f8b..a4fce9b 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -36,7 +36,7 @@ class CloudflareProvider(BaseProvider): ''' SUPPORTS_GEO = False # TODO: support SRV - SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'SPF', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SPF', 'TXT')) MIN_TTL = 120 TIMEOUT = 15 @@ -104,6 +104,20 @@ class CloudflareProvider(BaseProvider): 'values': [r['content'].replace(';', '\;') for r in records], } + def _data_for_CAA(self, _type, records): + values = [] + for r in records: + values.append({ + 'flags': r['flags'], + 'tag': r['tag'], + 'value': r['content'], + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + def _data_for_CNAME(self, _type, records): only = records[0] return { @@ -197,6 +211,14 @@ class CloudflareProvider(BaseProvider): _contents_for_NS = _contents_for_multiple _contents_for_SPF = _contents_for_multiple + def _contents_for_CAA(self, record): + for value in record.values: + yield { + 'flags': value.flags, + 'tag': value.tag, + 'value': value.value, + } + def _contents_for_TXT(self, record): for value in record.values: yield {'content': value.replace('\;', ';')} diff --git a/octodns/provider/dnsimple.py b/octodns/provider/dnsimple.py index dc44d1b..43b5b9b 100644 --- a/octodns/provider/dnsimple.py +++ b/octodns/provider/dnsimple.py @@ -91,8 +91,8 @@ class DnsimpleProvider(BaseProvider): account: 42 ''' SUPPORTS_GEO = False - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', - 'SPF', 'SRV', 'SSHFP', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', + 'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT')) def __init__(self, id, token, account, *args, **kwargs): self.log = logging.getLogger('DnsimpleProvider[{}]'.format(id)) @@ -114,6 +114,21 @@ class DnsimpleProvider(BaseProvider): _data_for_SPF = _data_for_multiple _data_for_TXT = _data_for_multiple + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + flags, tag, value = record['content'].split(' ') + values.append({ + 'flags': flags, + 'tag': tag, + 'value': value[1:-1], + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + def _data_for_CNAME(self, _type, records): record = records[0] return { @@ -275,6 +290,16 @@ class DnsimpleProvider(BaseProvider): _params_for_SPF = _params_for_multiple _params_for_TXT = _params_for_multiple + def _params_for_CAA(self, record): + for value in record.values: + yield { + 'content': '{} {} "{}"'.format(value.flags, value.tag, + value.value), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + def _params_for_single(self, record): yield { 'content': record.value, diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index 24c49d5..9800155 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -118,14 +118,33 @@ "meta": { "auto_added": false } + }, + { + "id": "fc223b34cd5611334422ab3322997667", + "type": "CAA", + "name": "unit.tests", + "content": "ca.unit.tests", + "flags": 0, + "tag": "issue", + "proxiable": false, + "proxied": false, + "ttl": 3600, + "locked": false, + "zone_id": "ff12ab34cd5611334422ab3322997650", + "zone_name": "unit.tests", + "modified_on": "2017-03-11T18:01:42.961566Z", + "created_on": "2017-03-11T18:01:42.961566Z", + "meta": { + "auto_added": false + } } ], "result_info": { "page": 2, "per_page": 10, "total_pages": 2, - "count": 7, - "total_count": 17 + "count": 8, + "total_count": 19 }, "success": true, "errors": [], diff --git a/tests/fixtures/dnsimple-page-2.json b/tests/fixtures/dnsimple-page-2.json index f50704b..40aaa48 100644 --- a/tests/fixtures/dnsimple-page-2.json +++ b/tests/fixtures/dnsimple-page-2.json @@ -159,12 +159,28 @@ "system_record": false, "created_at": "2017-03-09T15:55:09Z", "updated_at": "2017-03-09T15:55:09Z" + }, + { + "id": 12188803, + "zone_id": "unit.tests", + "parent_id": null, + "name": "", + "content": "0 issue \"ca.unit.tests\"", + "ttl": 3600, + "priority": null, + "type": "CAA", + "regions": [ + "global" + ], + "system_record": false, + "created_at": "2017-03-09T15:55:09Z", + "updated_at": "2017-03-09T15:55:09Z" } ], "pagination": { "current_page": 2, "per_page": 20, - "total_entries": 29, + "total_entries": 30, "total_pages": 2 } } diff --git a/tests/test_octodns_provider_cloudflare.py b/tests/test_octodns_provider_cloudflare.py index 5dcae30..04a46e0 100644 --- a/tests/test_octodns_provider_cloudflare.py +++ b/tests/test_octodns_provider_cloudflare.py @@ -118,7 +118,7 @@ class TestCloudflareProvider(TestCase): zone = Zone('unit.tests.', []) provider.populate(zone) - self.assertEquals(9, len(zone.records)) + self.assertEquals(10, len(zone.records)) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) @@ -126,7 +126,7 @@ class TestCloudflareProvider(TestCase): # re-populating the same zone/records comes out of cache, no calls again = Zone('unit.tests.', []) provider.populate(again) - self.assertEquals(9, len(again.records)) + self.assertEquals(10, len(again.records)) def test_apply(self): provider = CloudflareProvider('test', 'email', 'token') @@ -140,12 +140,12 @@ class TestCloudflareProvider(TestCase): 'id': 42, } }, # zone create - ] + [None] * 16 # individual record creates + ] + [None] * 17 # individual record creates # non-existant zone, create everything plan = provider.plan(self.expected) - self.assertEquals(9, len(plan.changes)) - self.assertEquals(9, provider.apply(plan)) + self.assertEquals(10, len(plan.changes)) + self.assertEquals(10, provider.apply(plan)) provider._request.assert_has_calls([ # created the domain @@ -170,7 +170,7 @@ class TestCloudflareProvider(TestCase): }), ], True) # expected number of total calls - self.assertEquals(18, provider._request.call_count) + self.assertEquals(19, provider._request.call_count) provider._request.reset_mock() From ba6dc9858e6e572d46f5b9d0d7778210b53c0410 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 28 Aug 2017 13:40:25 -0700 Subject: [PATCH 58/84] Get out of the business of validating CAA records Seem to be pretty inconsistently implemented/validated across providers so just shrug and move on. --- octodns/record.py | 2 +- tests/test_octodns_record.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/octodns/record.py b/octodns/record.py index aa41606..8ef80be 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -393,7 +393,7 @@ class CaaValue(object): reasons = [] try: flags = int(value.get('flags', 0)) - if flags not in (0, 128): + if flags < 0 or flags > 255: reasons.append('invalid flags "{}"'.format(flags)) except ValueError: reasons.append('invalid flags "{}"'.format(value['flags'])) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 215abe1..51676a3 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -939,12 +939,23 @@ class TestRecordValidation(TestCase): 'type': 'CAA', 'ttl': 600, 'value': { - 'flags': 42, + 'flags': -42, 'tag': 'iodef', 'value': 'http://foo.bar.com/', } }) - self.assertEquals(['invalid flags "42"'], ctx.exception.reasons) + self.assertEquals(['invalid flags "-42"'], ctx.exception.reasons) + with self.assertRaises(ValidationError) as ctx: + Record.new(self.zone, '', { + 'type': 'CAA', + 'ttl': 600, + 'value': { + 'flags': 442, + 'tag': 'iodef', + 'value': 'http://foo.bar.com/', + } + }) + self.assertEquals(['invalid flags "442"'], ctx.exception.reasons) with self.assertRaises(ValidationError) as ctx: Record.new(self.zone, '', { 'type': 'CAA', From a558fde6df1c2c0763fcfd22642d72441f9cb698 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 6 Sep 2017 12:08:08 -0700 Subject: [PATCH 59/84] Fixes for cloudflare CAA support --- octodns/provider/cloudflare.py | 15 +++++++-------- tests/fixtures/cloudflare-dns_records-page-2.json | 8 +++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/octodns/provider/cloudflare.py b/octodns/provider/cloudflare.py index a4fce9b..dd53b3a 100644 --- a/octodns/provider/cloudflare.py +++ b/octodns/provider/cloudflare.py @@ -107,11 +107,8 @@ class CloudflareProvider(BaseProvider): def _data_for_CAA(self, _type, records): values = [] for r in records: - values.append({ - 'flags': r['flags'], - 'tag': r['tag'], - 'value': r['content'], - }) + data = r['data'] + values.append(data) return { 'ttl': records[0]['ttl'], 'type': _type, @@ -214,9 +211,11 @@ class CloudflareProvider(BaseProvider): def _contents_for_CAA(self, record): for value in record.values: yield { - 'flags': value.flags, - 'tag': value.tag, - 'value': value.value, + 'data': { + 'flags': value.flags, + 'tag': value.tag, + 'value': value.value, + } } def _contents_for_TXT(self, record): diff --git a/tests/fixtures/cloudflare-dns_records-page-2.json b/tests/fixtures/cloudflare-dns_records-page-2.json index 9800155..195d6de 100644 --- a/tests/fixtures/cloudflare-dns_records-page-2.json +++ b/tests/fixtures/cloudflare-dns_records-page-2.json @@ -123,9 +123,11 @@ "id": "fc223b34cd5611334422ab3322997667", "type": "CAA", "name": "unit.tests", - "content": "ca.unit.tests", - "flags": 0, - "tag": "issue", + "data": { + "flags": 0, + "tag": "issue", + "value": "ca.unit.tests" + }, "proxiable": false, "proxied": false, "ttl": 3600, From ce7b2ef181adeda05353a776c5b52f50495e7a3a Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 6 Sep 2017 13:23:26 -0700 Subject: [PATCH 60/84] Cut v0.8.6 --- octodns/__init__.py | 2 +- script/release | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/octodns/__init__.py b/octodns/__init__.py index 601734c..bfb1905 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.8.5' +__VERSION__ = '0.8.6' diff --git a/script/release b/script/release index 16e7641..d8fabf2 100755 --- a/script/release +++ b/script/release @@ -8,5 +8,7 @@ ROOT=$(pwd) VERSION=$(grep __VERSION__ $ROOT/octodns/__init__.py | sed -e "s/.* = '//" -e "s/'$//") git tag -s v$VERSION -m "Release $VERSION" +git push origin v$VERSION +echo "Tagged and pushed v$VERSION" python setup.py sdist upload echo "Updloaded $VERSION" From 8a13ccab466c9ab9e641959238e1de18355f1f6d Mon Sep 17 00:00:00 2001 From: trnsnt Date: Mon, 18 Sep 2017 14:59:41 +0200 Subject: [PATCH 61/84] Add OVH as octodns provider --- README.md | 1 + octodns/provider/ovh.py | 322 ++++++++++++++++++++++++++ requirements.txt | 1 + tests/test_octodns_provider_ovh.py | 359 +++++++++++++++++++++++++++++ 4 files changed, 683 insertions(+) create mode 100644 octodns/provider/ovh.py create mode 100644 tests/test_octodns_provider_ovh.py diff --git a/README.md b/README.md index 1f103f1..afe92ac 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,7 @@ The above command pulled the existing data out of Route53 and placed the results | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | | [Ns1Provider](/octodns/provider/ns1.py) | All | No | | +| [OVH](/octodns/provider/ovh.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | | | [Route53](/octodns/provider/route53.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | | | [TinyDNSSource](/octodns/source/tinydns.py) | A, CNAME, MX, NS, PTR | No | read-only | diff --git a/octodns/provider/ovh.py b/octodns/provider/ovh.py new file mode 100644 index 0000000..b890862 --- /dev/null +++ b/octodns/provider/ovh.py @@ -0,0 +1,322 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import logging +from collections import defaultdict + +import ovh + +from octodns.record import Record +from .base import BaseProvider + + +class OvhProvider(BaseProvider): + """ + OVH provider using API v6 + + ovh: + class: octodns.provider.ovh.OvhProvider + # OVH api v6 endpoint + endpoint: ovh-eu + # API application key + application_key: 1234 + # API application secret + application_secret: 1234 + # API consumer key + consumer_key: 1234 + """ + + SUPPORTS_GEO = False + + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', + 'SRV', 'SSHFP', 'TXT')) + + def __init__(self, id, endpoint, application_key, application_secret, + consumer_key, *args, **kwargs): + self.log = logging.getLogger('OvhProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, endpoint=%s, application_key=%s, ' + 'application_secret=***, consumer_key=%s', id, endpoint, + application_key, consumer_key) + super(OvhProvider, self).__init__(id, *args, **kwargs) + self._client = ovh.Client( + endpoint=endpoint, + application_key=application_key, + application_secret=application_secret, + consumer_key=consumer_key, + ) + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + zone_name = zone.name[:-1] + records = self.get_records(zone_name=zone_name) + + values = defaultdict(lambda: defaultdict(list)) + for record in records: + values[record['subDomain']][record['fieldType']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record) + + self.log.info('populate: found %s records', + len(zone.records) - before) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + zone_name = desired.name[:-1] + self.log.info('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name).lower())(zone_name, + change) + + # We need to refresh the zone to really apply the changes + self._client.post('/domain/zone/{}/refresh'.format(zone_name)) + + def _apply_create(self, zone_name, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self.create_record(zone_name, params) + + def _apply_update(self, zone_name, change): + self._apply_delete(zone_name, change) + self._apply_create(zone_name, change) + + def _apply_delete(self, zone_name, change): + existing = change.existing + self.delete_records(zone_name, existing._type, existing.name) + + @staticmethod + def _data_for_multiple(_type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [record['target'] for record in records] + } + + @staticmethod + def _data_for_single(_type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': record['target'] + } + + @staticmethod + def _data_for_MX(_type, records): + values = [] + for record in records: + preference, exchange = record['target'].split(' ', 1) + values.append({ + 'preference': preference, + 'exchange': exchange, + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + + @staticmethod + def _data_for_NAPTR(_type, records): + values = [] + for record in records: + order, preference, flags, service, regexp, replacement = record[ + 'target'].split(' ', 5) + values.append({ + 'flags': flags[1:-1], + 'order': order, + 'preference': preference, + 'regexp': regexp[1:-1], + 'replacement': replacement, + 'service': service[1:-1], + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + @staticmethod + def _data_for_SRV(_type, records): + values = [] + for record in records: + priority, weight, port, target = record['target'].split(' ', 3) + values.append({ + 'port': port, + 'priority': priority, + 'target': '{}.'.format(target), + 'weight': weight + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + @staticmethod + def _data_for_SSHFP(_type, records): + values = [] + for record in records: + algorithm, fingerprint_type, fingerprint = record['target'].split( + ' ', 2) + values.append({ + 'algorithm': algorithm, + 'fingerprint': fingerprint, + 'fingerprint_type': fingerprint_type + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + _data_for_NS = _data_for_multiple + _data_for_TXT = _data_for_multiple + _data_for_SPF = _data_for_multiple + _data_for_PTR = _data_for_single + _data_for_CNAME = _data_for_single + + @staticmethod + def _params_for_multiple(record): + for value in record.values: + yield { + 'target': value, + 'subDomain': record.name, + 'ttl': record.ttl, + 'fieldType': record._type, + } + + @staticmethod + def _params_for_single(record): + yield { + 'target': record.value, + 'subDomain': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + + @staticmethod + def _params_for_MX(record): + for value in record.values: + yield { + 'target': '%d %s' % (value.preference, value.exchange), + 'subDomain': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + + @staticmethod + def _params_for_NAPTR(record): + for value in record.values: + content = '{} {} "{}" "{}" "{}" {}' \ + .format(value.order, value.preference, value.flags, + value.service, value.regexp, value.replacement) + yield { + 'target': content, + 'subDomain': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + + @staticmethod + def _params_for_SRV(record): + for value in record.values: + yield { + 'subDomain': '{} {} {} {}'.format(value.priority, + value.weight, value.port, + value.target), + 'target': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + + @staticmethod + def _params_for_SSHFP(record): + for value in record.values: + yield { + 'subDomain': '{} {} {}'.format(value.algorithm, + value.fingerprint_type, + value.fingerprint), + 'target': record.name, + 'ttl': record.ttl, + 'fieldType': record._type + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + _params_for_SPF = _params_for_multiple + _params_for_TXT = _params_for_multiple + + _params_for_CNAME = _params_for_single + _params_for_PTR = _params_for_single + + def get_records(self, zone_name): + """ + List all records of a DNS zone + :param zone_name: Name of zone + :return: list of id's records + """ + records = self._client.get('/domain/zone/{}/record'.format(zone_name)) + return [self.get_record(zone_name, record_id) for record_id in records] + + def get_record(self, zone_name, record_id): + """ + Get record with given id + :param zone_name: Name of the zone + :param record_id: Id of the record + :return: Value of the record + """ + return self._client.get( + '/domain/zone/{}/record/{}'.format(zone_name, record_id)) + + def delete_records(self, zone_name, record_type, subdomain): + """ + Delete record from have fieldType=type and subDomain=subdomain + :param zone_name: Name of the zone + :param record_type: fieldType + :param subdomain: subDomain + """ + records = self._client.get('/domain/zone/{}/record'.format(zone_name), + fieldType=record_type, subDomain=subdomain) + for record in records: + self.delete_record(zone_name, record) + + def delete_record(self, zone_name, record_id): + """ + Delete record with a given id + :param zone_name: Name of the zone + :param record_id: Id of the record + """ + self.log.debug('Delete record: zone: %s, id %s', zone_name, + record_id) + self._client.delete( + '/domain/zone/{}/record/{}'.format(zone_name, record_id)) + + def create_record(self, zone_name, params): + """ + Create a record + :param zone_name: Name of the zone + :param params: {'fieldType': 'A', 'ttl': 60, 'subDomain': 'www', + 'target': '1.2.3.4' + """ + self.log.debug('Create record: zone: %s, id %s', zone_name, + params) + return self._client.post('/domain/zone/{}/record'.format(zone_name), + **params) diff --git a/requirements.txt b/requirements.txt index d2be70f..a7d1d94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ jmespath==0.9.3 msrestazure==0.4.10 natsort==5.0.3 nsone==0.9.14 +ovh==0.4.7 python-dateutil==2.6.1 requests==2.13.0 s3transfer==0.1.10 diff --git a/tests/test_octodns_provider_ovh.py b/tests/test_octodns_provider_ovh.py new file mode 100644 index 0000000..2816748 --- /dev/null +++ b/tests/test_octodns_provider_ovh.py @@ -0,0 +1,359 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from unittest import TestCase + +from mock import patch, call +from ovh import APIError + +from octodns.provider.ovh import OvhProvider +from octodns.record import Record +from octodns.zone import Zone + + +class TestOvhProvider(TestCase): + api_record = [] + + zone = Zone('unit.tests.', []) + expected = set() + + # A, subdomain='' + api_record.append({ + 'fieldType': 'A', + 'ttl': 100, + 'target': '1.2.3.4', + 'subDomain': '', + 'id': 1 + }) + expected.add(Record.new(zone, '', { + 'ttl': 100, + 'type': 'A', + 'value': '1.2.3.4', + })) + + # A, subdomain='sub + api_record.append({ + 'fieldType': 'A', + 'ttl': 200, + 'target': '1.2.3.4', + 'subDomain': 'sub', + 'id': 2 + }) + expected.add(Record.new(zone, 'sub', { + 'ttl': 200, + 'type': 'A', + 'value': '1.2.3.4', + })) + + # CNAME + api_record.append({ + 'fieldType': 'CNAME', + 'ttl': 300, + 'target': 'unit.tests.', + 'subDomain': 'www2', + 'id': 3 + }) + expected.add(Record.new(zone, 'www2', { + 'ttl': 300, + 'type': 'CNAME', + 'value': 'unit.tests.', + })) + + # MX + api_record.append({ + 'fieldType': 'MX', + 'ttl': 400, + 'target': '10 mx1.unit.tests.', + 'subDomain': '', + 'id': 4 + }) + expected.add(Record.new(zone, '', { + 'ttl': 400, + 'type': 'MX', + 'values': [{ + 'preference': 10, + 'exchange': 'mx1.unit.tests.', + }] + })) + + # NAPTR + api_record.append({ + 'fieldType': 'NAPTR', + 'ttl': 500, + 'target': '10 100 "S" "SIP+D2U" "!^.*$!sip:info@bar.example.com!" .', + 'subDomain': 'naptr', + 'id': 5 + }) + expected.add(Record.new(zone, 'naptr', { + 'ttl': 500, + 'type': 'NAPTR', + 'values': [{ + 'flags': 'S', + 'order': 10, + 'preference': 100, + 'regexp': '!^.*$!sip:info@bar.example.com!', + 'replacement': '.', + 'service': 'SIP+D2U', + }] + })) + + # NS + api_record.append({ + 'fieldType': 'NS', + 'ttl': 600, + 'target': 'ns1.unit.tests.', + 'subDomain': '', + 'id': 6 + }) + api_record.append({ + 'fieldType': 'NS', + 'ttl': 600, + 'target': 'ns2.unit.tests.', + 'subDomain': '', + 'id': 7 + }) + expected.add(Record.new(zone, '', { + 'ttl': 600, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.'], + })) + + # NS with sub + api_record.append({ + 'fieldType': 'NS', + 'ttl': 700, + 'target': 'ns3.unit.tests.', + 'subDomain': 'www3', + 'id': 8 + }) + api_record.append({ + 'fieldType': 'NS', + 'ttl': 700, + 'target': 'ns4.unit.tests.', + 'subDomain': 'www3', + 'id': 9 + }) + expected.add(Record.new(zone, 'www3', { + 'ttl': 700, + 'type': 'NS', + 'values': ['ns3.unit.tests.', 'ns4.unit.tests.'], + })) + + api_record.append({ + 'fieldType': 'SRV', + 'ttl': 800, + 'target': '10 20 30 foo-1.unit.tests.', + 'subDomain': '_srv._tcp', + 'id': 10 + }) + api_record.append({ + 'fieldType': 'SRV', + 'ttl': 800, + 'target': '40 50 60 foo-2.unit.tests.', + 'subDomain': '_srv._tcp', + 'id': 11 + }) + expected.add(Record.new(zone, '_srv._tcp', { + 'ttl': 800, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 40, + 'weight': 50, + 'port': 60, + 'target': 'foo-2.unit.tests.', + }] + })) + + # PTR + api_record.append({ + 'fieldType': 'PTR', + 'ttl': 900, + 'target': 'unit.tests.', + 'subDomain': '4', + 'id': 12 + }) + expected.add(Record.new(zone, '4', { + 'ttl': 900, + 'type': 'PTR', + 'value': 'unit.tests.' + })) + + # SPF + api_record.append({ + 'fieldType': 'SPF', + 'ttl': 1000, + 'target': 'v=spf1 include:unit.texts.rerirect ~all', + 'subDomain': '', + 'id': 13 + }) + expected.add(Record.new(zone, '', { + 'ttl': 1000, + 'type': 'SPF', + 'value': 'v=spf1 include:unit.texts.rerirect ~all' + })) + + # SSHFP + api_record.append({ + 'fieldType': 'SSHFP', + 'ttl': 1100, + 'target': '1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73 ', + 'subDomain': '', + 'id': 14 + }) + expected.add(Record.new(zone, '', { + 'ttl': 1100, + 'type': 'SSHFP', + 'value': { + 'algorithm': 1, + 'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73', + 'fingerprint_type': 1 + } + })) + + # AAAA + api_record.append({ + 'fieldType': 'AAAA', + 'ttl': 1200, + 'target': '1:1ec:1::1', + 'subDomain': '', + 'id': 15 + }) + expected.add(Record.new(zone, '', { + 'ttl': 200, + 'type': 'AAAA', + 'value': '1:1ec:1::1', + })) + + @patch('ovh.Client') + def test_populate(self, client_mock): + provider = OvhProvider('test', 'endpoint', 'application_key', + 'application_secret', 'consumer_key') + + with patch.object(provider._client, 'get') as get_mock: + zone = Zone('unit.tests.', []) + get_mock.side_effect = APIError('boom') + with self.assertRaises(APIError) as ctx: + provider.populate(zone) + self.assertEquals(get_mock.side_effect, ctx.exception) + + with patch.object(provider._client, 'get') as get_mock: + zone = Zone('unit.tests.', []) + get_returns = [[record['id'] for record in self.api_record]] + get_returns += self.api_record + get_mock.side_effect = get_returns + provider.populate(zone) + self.assertEquals(self.expected, zone.records) + + @patch('ovh.Client') + def test_apply(self, client_mock): + provider = OvhProvider('test', 'endpoint', 'application_key', + 'application_secret', 'consumer_key') + + desired = Zone('unit.tests.', []) + + for r in self.expected: + desired.add_record(r) + + with patch.object(provider._client, 'post') as get_mock: + plan = provider.plan(desired) + get_mock.side_effect = APIError('boom') + with self.assertRaises(APIError) as ctx: + provider.apply(plan) + self.assertEquals(get_mock.side_effect, ctx.exception) + + with patch.object(provider._client, 'get') as get_mock: + get_returns = [[1, 2], { + 'fieldType': 'A', + 'ttl': 600, + 'target': '5.6.7.8', + 'subDomain': '', + 'id': 100 + }, {'fieldType': 'A', + 'ttl': 600, + 'target': '5.6.7.8', + 'subDomain': 'fake', + 'id': 101 + }] + get_mock.side_effect = get_returns + + plan = provider.plan(desired) + + with patch.object(provider._client, 'post') as post_mock: + with patch.object(provider._client, 'delete') as delete_mock: + with patch.object(provider._client, 'get') as get_mock: + get_mock.side_effect = [[100], [101]] + provider.apply(plan) + wanted_calls = [ + call(u'/domain/zone/unit.tests/record', + fieldType=u'A', + subDomain=u'', target=u'1.2.3.4', ttl=100), + call(u'/domain/zone/unit.tests/record', + fieldType=u'SRV', + subDomain=u'10 20 30 foo-1.unit.tests.', + target='_srv._tcp', ttl=800), + call(u'/domain/zone/unit.tests/record', + fieldType=u'SRV', + subDomain=u'40 50 60 foo-2.unit.tests.', + target='_srv._tcp', ttl=800), + call(u'/domain/zone/unit.tests/record', + fieldType=u'PTR', subDomain='4', + target=u'unit.tests.', ttl=900), + call(u'/domain/zone/unit.tests/record', + fieldType=u'NS', subDomain='www3', + target=u'ns3.unit.tests.', ttl=700), + call(u'/domain/zone/unit.tests/record', + fieldType=u'NS', subDomain='www3', + target=u'ns4.unit.tests.', ttl=700), + call(u'/domain/zone/unit.tests/record', + fieldType=u'SSHFP', + subDomain=u'1 1 bf6b6825d2977c511a475bbefb88a' + u'ad54' + u'a92ac73', + target=u'', ttl=1100), + call(u'/domain/zone/unit.tests/record', + fieldType=u'AAAA', subDomain=u'', + target=u'1:1ec:1::1', ttl=200), + call(u'/domain/zone/unit.tests/record', + fieldType=u'MX', subDomain=u'', + target=u'10 mx1.unit.tests.', ttl=400), + call(u'/domain/zone/unit.tests/record', + fieldType=u'CNAME', subDomain='www2', + target=u'unit.tests.', ttl=300), + call(u'/domain/zone/unit.tests/record', + fieldType=u'SPF', subDomain=u'', + target=u'v=spf1 include:unit.texts.' + u'rerirect ~all', + ttl=1000), + call(u'/domain/zone/unit.tests/record', + fieldType=u'A', + subDomain='sub', target=u'1.2.3.4', ttl=200), + call(u'/domain/zone/unit.tests/record', + fieldType=u'NAPTR', subDomain='naptr', + target=u'10 100 "S" "SIP+D2U" "!^.*$!sip:' + u'info@bar' + u'.example.com!" .', + ttl=500), + call(u'/domain/zone/unit.tests/refresh')] + + post_mock.assert_has_calls(wanted_calls) + + # Get for delete calls + get_mock.assert_has_calls( + [call(u'/domain/zone/unit.tests/record', + fieldType=u'A', subDomain=u''), + call(u'/domain/zone/unit.tests/record', + fieldType=u'A', subDomain='fake')] + ) + # 2 delete calls, one for update + one for delete + delete_mock.assert_has_calls( + [call(u'/domain/zone/unit.tests/record/100'), + call(u'/domain/zone/unit.tests/record/101')]) From feec443bb4f1127673c738869251032066a7ef76 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 29 Sep 2017 16:14:22 -0700 Subject: [PATCH 62/84] Require setuptools new enough to publish to pypi --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 265f407..5cdf252 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,4 @@ nose pep8 pyflakes requests_mock +setuptools>=36.4.0 From 74631a6a71779f59be50daed27044e11a314cbc6 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 29 Sep 2017 16:23:51 -0700 Subject: [PATCH 63/84] 0.8.7 version bump --- CHANGELOG.md | 13 +++++++++++++ octodns/__init__.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d58d4de..9c97cdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## v0.8.7 - 2017-09-29 - OVH support + +Adds an OVH provider. + +## v0.8.6 - 2017-09-06 - CAA record type, + +Misc fixes and improvments. + +* Azure TXT record fix +* PowerDNS api support for https +* Configurable Route53 max retries and max-attempts +* Improved key ordering error message + ## v0.8.5 - 2017-07-21 - Azure, NS1 escaping, & large zones Relatively small delta this go around. No major themes or anything, just steady diff --git a/octodns/__init__.py b/octodns/__init__.py index bfb1905..3740dec 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -3,4 +3,4 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -__VERSION__ = '0.8.6' +__VERSION__ = '0.8.7' From 70120bedc82aca9103aacb4b25dc12a379b9a65d Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 5 Oct 2017 10:04:29 -0700 Subject: [PATCH 64/84] Implement "chunked" TXT/SPF value support for long values This implements it transparently at Record level. Providers that need things to be chunked (seems to just be Route53 an Dyn) switch to use `chunked_values`, but everything else can stick with `values`. I've run through each provider I have access to verifying that things operate as expected/required. OVH and Azure are untested. --- octodns/provider/dyn.py | 2 +- octodns/provider/route53.py | 3 +- octodns/record.py | 35 ++++++++++++++-------- tests/test_octodns_record.py | 56 ++++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 15 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 3b7b9ea..721b3a7 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -455,7 +455,7 @@ class DynProvider(BaseProvider): return [{ 'txtdata': v, 'ttl': record.ttl, - } for v in record.values] + } for v in record.chunked_values] def _kwargs_for_SRV(self, record): return [{ diff --git a/octodns/provider/route53.py b/octodns/provider/route53.py index 0600511..7623648 100644 --- a/octodns/provider/route53.py +++ b/octodns/provider/route53.py @@ -114,8 +114,7 @@ class _Route53Record(object): for v in record.values] def _values_for_quoted(self, record): - return ['"{}"'.format(v.replace('"', '\\"')) - for v in record.values] + return record.chunked_values _values_for_SPF = _values_for_quoted _values_for_TXT = _values_for_quoted diff --git a/octodns/record.py b/octodns/record.py index 8ef80be..554d98b 100644 --- a/octodns/record.py +++ b/octodns/record.py @@ -704,8 +704,8 @@ class SshfpRecord(_ValuesMixin, Record): _unescaped_semicolon_re = re.compile(r'\w;') -class SpfRecord(_ValuesMixin, Record): - _type = 'SPF' +class _ChunkedValuesMixin(_ValuesMixin): + CHUNK_SIZE = 255 @classmethod def _validate_value(cls, value): @@ -714,9 +714,29 @@ class SpfRecord(_ValuesMixin, Record): return [] def _process_values(self, values): + ret = [] + for v in values: + if v and v[0] == '"': + v = v[1:-1] + ret.append(v.replace('" "', '')) + return ret + + @property + def chunked_values(self): + values = [] + for v in self.values: + v = v.replace('"', '\\"') + vs = [v[i:i + self.CHUNK_SIZE] + for i in range(0, len(v), self.CHUNK_SIZE)] + vs = '" "'.join(vs) + values.append('"{}"'.format(vs)) return values +class SpfRecord(_ChunkedValuesMixin, Record): + _type = 'SPF' + + class SrvValue(object): @classmethod @@ -797,14 +817,5 @@ class SrvRecord(_ValuesMixin, Record): return [SrvValue(v) for v in values] -class TxtRecord(_ValuesMixin, Record): +class TxtRecord(_ChunkedValuesMixin, Record): _type = 'TXT' - - @classmethod - def _validate_value(cls, value): - if _unescaped_semicolon_re.search(value): - return ['unescaped ;'] - return [] - - def _process_values(self, values): - return values diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 51676a3..41b63a9 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -1490,3 +1490,59 @@ class TestRecordValidation(TestCase): 'value': 'this has some; semi-colons\; in it', }) self.assertEquals(['unescaped ;'], ctx.exception.reasons) + + def test_TXT_long_value_chunking(self): + expected = '"Lorem ipsum dolor sit amet, consectetur adipiscing ' \ + 'elit, seddo eiusmod tempor incididunt ut labore et dolore ' \ + 'magna aliqua. Ut enim ad minim veniam, quis nostrud ' \ + 'exercitation ullamco laboris nisi ut aliquip ex ea commodo ' \ + 'consequat. Duis aute irure dolor in" " reprehenderit in ' \ + 'voluptate velit esse cillum dolore eu fugiat nulla pariatur. ' \ + 'Excepteur sint occaecat cupidatat non proident, sunt in culpa ' \ + 'qui officia deserunt mollit anim id est laborum."' + + # Single string + single = Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + 'values': [ + 'hello world', + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed' + 'do eiusmod tempor incididunt ut labore et dolore magna ' + 'aliqua. Ut enim ad minim veniam, quis nostrud exercitation ' + 'ullamco laboris nisi ut aliquip ex ea commodo consequat. ' + 'Duis aute irure dolor in reprehenderit in voluptate velit ' + 'esse cillum dolore eu fugiat nulla pariatur. Excepteur sint ' + 'occaecat cupidatat non proident, sunt in culpa qui officia ' + 'deserunt mollit anim id est laborum.', + 'this has some\; semi-colons\; in it', + ] + }) + self.assertEquals(3, len(single.values)) + self.assertEquals(3, len(single.chunked_values)) + # Note we are checking that this normalizes the chunking, not that we + # get out what we put in. + self.assertEquals(expected, single.chunked_values[0]) + + # Chunked + chunked = Record.new(self.zone, '', { + 'type': 'TXT', + 'ttl': 600, + 'values': [ + '"hello world"', + '"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed' + 'do eiusmod tempor incididunt ut labore et dolore magna ' + 'aliqua. Ut enim ad minim veniam, quis nostrud exercitation ' + 'ullamco laboris nisi ut aliquip ex" " ea commodo consequat. ' + 'Duis aute irure dolor in reprehenderit in voluptate velit ' + 'esse cillum dolore eu fugiat nulla pariatur. Excepteur sint ' + 'occaecat cupidatat non proident, sunt in culpa qui officia ' + 'deserunt mollit anim id est laborum."', + '"this has some\; semi-colons\; in it"', + ] + }) + self.assertEquals(expected, chunked.chunked_values[0]) + # should be single values, no quoting + self.assertEquals(single.values, chunked.values) + # should be chunked values, with quoting + self.assertEquals(single.chunked_values, chunked.chunked_values) From 30efda329559cf82eb7666a3ccd5fdcda6be986e Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Mon, 9 Oct 2017 09:00:15 -0700 Subject: [PATCH 65/84] Make long TXT record concat cleaerer --- tests/test_octodns_record.py | 40 ++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/tests/test_octodns_record.py b/tests/test_octodns_record.py index 41b63a9..46a5e65 100644 --- a/tests/test_octodns_record.py +++ b/tests/test_octodns_record.py @@ -1493,28 +1493,30 @@ class TestRecordValidation(TestCase): def test_TXT_long_value_chunking(self): expected = '"Lorem ipsum dolor sit amet, consectetur adipiscing ' \ - 'elit, seddo eiusmod tempor incididunt ut labore et dolore ' \ + 'elit, sed do eiusmod tempor incididunt ut labore et dolore ' \ 'magna aliqua. Ut enim ad minim veniam, quis nostrud ' \ 'exercitation ullamco laboris nisi ut aliquip ex ea commodo ' \ - 'consequat. Duis aute irure dolor in" " reprehenderit in ' \ + 'consequat. Duis aute irure dolor i" "n reprehenderit in ' \ 'voluptate velit esse cillum dolore eu fugiat nulla pariatur. ' \ 'Excepteur sint occaecat cupidatat non proident, sunt in culpa ' \ 'qui officia deserunt mollit anim id est laborum."' + long_value = 'Lorem ipsum dolor sit amet, consectetur adipiscing ' \ + 'elit, sed do eiusmod tempor incididunt ut labore et dolore ' \ + 'magna aliqua. Ut enim ad minim veniam, quis nostrud ' \ + 'exercitation ullamco laboris nisi ut aliquip ex ea commodo ' \ + 'consequat. Duis aute irure dolor in reprehenderit in ' \ + 'voluptate velit esse cillum dolore eu fugiat nulla ' \ + 'pariatur. Excepteur sint occaecat cupidatat non proident, ' \ + 'sunt in culpa qui officia deserunt mollit anim id est ' \ + 'laborum.' # Single string single = Record.new(self.zone, '', { 'type': 'TXT', 'ttl': 600, 'values': [ 'hello world', - 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed' - 'do eiusmod tempor incididunt ut labore et dolore magna ' - 'aliqua. Ut enim ad minim veniam, quis nostrud exercitation ' - 'ullamco laboris nisi ut aliquip ex ea commodo consequat. ' - 'Duis aute irure dolor in reprehenderit in voluptate velit ' - 'esse cillum dolore eu fugiat nulla pariatur. Excepteur sint ' - 'occaecat cupidatat non proident, sunt in culpa qui officia ' - 'deserunt mollit anim id est laborum.', + long_value, 'this has some\; semi-colons\; in it', ] }) @@ -1524,20 +1526,22 @@ class TestRecordValidation(TestCase): # get out what we put in. self.assertEquals(expected, single.chunked_values[0]) + long_split_value = '"Lorem ipsum dolor sit amet, consectetur ' \ + 'adipiscing elit, sed do eiusmod tempor incididunt ut ' \ + 'labore et dolore magna aliqua. Ut enim ad minim veniam, ' \ + 'quis nostrud exercitation ullamco laboris nisi ut aliquip ' \ + 'ex" " ea commodo consequat. Duis aute irure dolor in ' \ + 'reprehenderit in voluptate velit esse cillum dolore eu ' \ + 'fugiat nulla pariatur. Excepteur sint occaecat cupidatat ' \ + 'non proident, sunt in culpa qui officia deserunt mollit ' \ + 'anim id est laborum."' # Chunked chunked = Record.new(self.zone, '', { 'type': 'TXT', 'ttl': 600, 'values': [ '"hello world"', - '"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed' - 'do eiusmod tempor incididunt ut labore et dolore magna ' - 'aliqua. Ut enim ad minim veniam, quis nostrud exercitation ' - 'ullamco laboris nisi ut aliquip ex" " ea commodo consequat. ' - 'Duis aute irure dolor in reprehenderit in voluptate velit ' - 'esse cillum dolore eu fugiat nulla pariatur. Excepteur sint ' - 'occaecat cupidatat non proident, sunt in culpa qui officia ' - 'deserunt mollit anim id est laborum."', + long_split_value, '"this has some\; semi-colons\; in it"', ] }) From 6f0a12ae01a6fb0f0368264bcf48f8cc9089658d Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Mon, 2 Oct 2017 14:16:29 +0200 Subject: [PATCH 66/84] Add .idea (intelliJ) files to gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c45a684..1bca124 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ output/ tmp/ build/ config/ +.idea/ \ No newline at end of file From ed783b5ff2899828d2480a63cecf55ed04a9e6e7 Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Mon, 2 Oct 2017 14:17:09 +0200 Subject: [PATCH 67/84] Add proposed google cloud provider. Proposed google cloud provider for #23 --- octodns/provider/googlecloud.py | 363 ++++++++++++++++++ requirements.txt | 1 + script/test | 1 + tests/test_octodns_provider_googlecloud.py | 421 +++++++++++++++++++++ 4 files changed, 786 insertions(+) create mode 100644 octodns/provider/googlecloud.py create mode 100644 tests/test_octodns_provider_googlecloud.py diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py new file mode 100644 index 0000000..dff11fc --- /dev/null +++ b/octodns/provider/googlecloud.py @@ -0,0 +1,363 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +import re +import shlex +import time +from logging import getLogger + +from google.cloud import dns + +from .base import BaseProvider +from ..record import Record + + +class _GoogleCloudRecordSetMaker(object): + """Wrapper to make google cloud client resource record sets from OctoDNS + Records. + + googlecloud.py: + class: octodns.provider.googlecloud._GoogleCloudRecordSetMaker + An _GoogleCloudRecordSetMaker creates google cloued client resource + records which can be used to update the Google Cloud DNS zones. + """ + + def __init__(self, gcloud_zone, record): + self.gcloud_zone = gcloud_zone + self.record = record + + self._record_set_func = getattr( + self, '_record_set_from_{}'.format(record._type)) + + def get_record_set(self): + return self._record_set_func(self.record) + + def _record_set_from_A(self, record): + return self.gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, record.values) + + _record_set_from_AAAA = _record_set_from_A + + def _record_set_from_CAA(self, record): + return self.gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{flags} {tag} {value}'.format(**record.data['value'])]) + + def _record_set_from_CNAME(self, record): + return self.gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [record.value]) + + def _record_set_from_MX(self, record): + return self.gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{preference} {exchange}'.format(**v.data) + for v in record.values]) + + def _record_set_from_NAPTR(self, record): + return self.gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{order} {preference} "{flags}" "{service}" ' + '"{regexp}" {replacement}' + .format(**v.data) for v in record.values]) + + _record_set_from_NS = _record_set_from_A + + _record_set_from_PTR = _record_set_from_CNAME + + _record_set_from_SPF = _record_set_from_A + + def _record_set_from_SRV(self, record): + return self.gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{priority} {weight} {port} {target}' + .format(**v.data) for v in record.values]) + + def _record_set_from_TXT(self, record): + if 'values' in record.data: + val = record.data['values'] + else: + val = [record.data['value']] + + return self.gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, val) + + +class GoogleCloudProvider(BaseProvider): + """ + Google Cloud DNS provider + + google_cloud: + class: octodns.provider.googlecloud.GoogleCloudProvider + # Credentials file for a service_account or other account can be + # specified with the GOOGLE_APPLICATION_CREDENTIALS environment + # variable. (https://console.cloud.google.com/apis/credentials) + # + # The project to work on (not required) + # project: foobar + """ + + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NAPTR', + 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) + SUPPORTS_GEO = False + + def __init__(self, id, project=None, *args, **kwargs): + + # Logger + self.log = getLogger('GoogleCloudProvider[{}]'.format(id)) + self.id = id + + super(GoogleCloudProvider, self).__init__(id, *args, **kwargs) + self.gcloud_client = dns.Client(project=project) + + def _apply(self, plan): + """Required function of manager.py to actually apply a record change. + + :param plan: Contains the zones and changes to be made + :type plan: octodns.provider.base.Plan + + :type return: void + """ + desired = plan.desired + changes = plan.changes + + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + # Get gcloud zone, or create one if none existed before. + gcloud_zone = self._get_gcloud_zone(desired.name, create=True) + + gcloud_changes = gcloud_zone.changes() + + for change in changes: + class_name = change.__class__.__name__ + if class_name in 'Create': + gcloud_changes.add_record_set( + self._record_to_record_set(gcloud_zone, change.record)) + elif class_name == 'Delete': + gcloud_changes.delete_record_set( + self._record_to_record_set(gcloud_zone, change.record)) + elif class_name == 'Update': + gcloud_changes.delete_record_set( + self._record_to_record_set(gcloud_zone, change.existing)) + gcloud_changes.add_record_set( + self._record_to_record_set(gcloud_zone, change.new)) + else: + raise RuntimeError('Change type "{}" for change "{!s}" ' + 'is none of "Create", "Delete" or "Update' + .format(class_name, change)) + + gcloud_changes.create() + i = 1 + while gcloud_changes.status != 'done': + self.log.debug("Waiting for changes to complete") + time.sleep(i) + gcloud_changes.reload() + if i < 30: + i += 2 + + def _create_gcloud_zone(self, dns_name): + """Creates a google cloud ManagedZone with dns_name, and zone named + derived from it. calls .create() method and returns it. + + :param dns_name: fqdn of zone to create + :type dns_name: str + + :type return: new google.cloud.dns.ManagedZone + """ + # Zone name must begin with a letter, end with a letter or digit, + # and only contain lowercase letters, digits or dashes + zone_name = re.sub("[^a-z0-9-]", "", + dns_name[:-1].replace('.', "-")) + # make sure that the end result did not end up wo leading letter + if re.match('[^a-z]', zone_name[0]): + # I cannot think of a situation where a zone name derived from + # a domain name would'nt start with leading letter and thereby + # violate the constraint, however if such a situation is + # encountered, add a leading "a" here. + zone_name = "a%s" % zone_name + + gcloud_zone = self.gcloud_client.zone( + name=zone_name, + dns_name=dns_name + ) + gcloud_zone.create(client=self.gcloud_client) + + self.log.info("Created zone %s. Fqdn %s." % + (zone_name, dns_name)) + + return gcloud_zone + + def _get_gcloud_records(self, gcloud_zone, page_token=None): + """ Generator function which yields ResourceRecordSet for the managed + gcloud zone, until there are no more records to pull. + + :param gcloud_zone: zone to pull records from + :type gcloud_zone: google.cloud.dns.ManagedZone + :param page_token: page token for the page to get + + :return: a resource record set + :type return: google.cloud.dns.ResourceRecordSet + """ + gcloud_iterator = gcloud_zone.list_resource_record_sets( + page_token=page_token) + for gcloud_record in gcloud_iterator: + yield gcloud_record + # This is to get results which may be on a "paged" page. + # (if more than max_results) entries. + if gcloud_iterator.next_page_token: + for gcloud_record in self._get_gcloud_records( + gcloud_zone, gcloud_iterator.next_page_token): + # yield from is in python 3 only. + yield gcloud_record + + def _get_gcloud_zone(self, dns_name, page_token=None, create=False): + """Return the ManagedZone which has has the matching dns_name, or + None if no such zone exist, unless create=True, then create a new + one and return it. + + :param dns_name: fqdn of dns name for zone to get. + :type dns_name: str + :param page_token: page token for the page to get + :type page_token: str + :param create: if true, create ManagedZone if it does not exist + already + + :type return: new google.cloud.dns.ManagedZone + """ + # Find the google name for the incoming zone + gcloud_zones = self.gcloud_client.list_zones(page_token=page_token) + for gcloud_zone in gcloud_zones: + if gcloud_zone.dns_name == dns_name: + return gcloud_zone + else: + # Zone not found. Check if there are more results which could be + # retrieved by checking "next_page_token". + if gcloud_zones.next_page_token: + return self._get_gcloud_zone(dns_name, + gcloud_zones.next_page_token) + else: + # Nothing found, either return None or else create zone and + # return that one (if create=True) + self.log.debug('_get_gcloud_zone: zone name=%s, ' + 'was not found by %s.', + dns_name, self.gcloud_client) + if create: + return self._create_gcloud_zone(dns_name) + + @staticmethod + def _record_to_record_set(gcloud_zone, record): + """create google.cloud.dns.ResourceRecordSet from ocdodns.Record + + :param record: a record object + :type record: ocdodns.Record + :param gcloud_zone: a google gcloud zone + :type gcloud_zone: google.cloud.dns.ManagedZone + :type return: google.cloud.dns.ResourceRecordSet + """ + grm = _GoogleCloudRecordSetMaker(gcloud_zone, record) + + return grm.get_record_set() + + def populate(self, zone, target=False, lenient=False): + """Required function of manager.py to collect records from zone. + + :param zone: A dns zone + :type zone: octodns.zone.Zone + :param target: Unused. + :type target: bool + :param lenient: Unused. Check octodns.manager for usage. + :type lenient: bool + + :type return: void + """ + + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + before = len(zone.records) + + gcloud_zone = self._get_gcloud_zone(zone.name) + + _records = set() + if gcloud_zone: + for gcloud_record in self._get_gcloud_records(gcloud_zone): + if gcloud_record.record_type.upper() in self.SUPPORTS: + _records.add(gcloud_record) + for gcloud_record in _records: + record_name = gcloud_record.name + if record_name.endswith(zone.name): + # google cloud always return fqdn. Make relative record + # here. "root" records will then get the '' record_name, + # which is also the way dyn likes it. + record_name = record_name[:-(len(zone.name) + 1)] + typ = gcloud_record.record_type.upper() + data = getattr(self, '_data_for_{}'.format(typ)) + data = data(gcloud_record) + data['type'] = typ + data['ttl'] = gcloud_record.ttl + self.log.debug('populate: adding record {} records: {!s}' + .format(record_name, data)) + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) + + self.log.info('populate: found %s records', len(zone.records) - before) + + def _data_for_A(self, gcloud_record): + return { + 'values': gcloud_record.rrdatas + } + + _data_for_AAAA = _data_for_A + + def _data_for_CAA(self, gcloud_record): + return { + 'values': [{ + 'flags': v[0], + 'tag': v[1], + 'value': v[2]} + for v in [shlex.split(g) for g in gcloud_record.rrdatas]]} + + def _data_for_CNAME(self, gcloud_record): + return { + 'value': gcloud_record.rrdatas[0] + } + + def _data_for_MX(self, gcloud_record): + return {'values': [{ + "preference": v[0], + "exchange": v[1]} + for v in [shlex.split(g) for g in gcloud_record.rrdatas]]} + + def _data_for_NAPTR(self, gcloud_record): + return {'values': [{ + 'order': v[0], + 'preference': v[1], + 'flags': v[2], + 'service': v[3], + 'regexp': v[4], + 'replacement': v[5]} + for v in [shlex.split(g) for g in gcloud_record.rrdatas]]} + + _data_for_NS = _data_for_A + + _data_for_PTR = _data_for_CNAME + + _data_for_SPF = _data_for_A + + def _data_for_SRV(self, gcloud_record): + return {'values': [{ + 'priority': v[0], + 'weight': v[1], + 'port': v[2], + 'target': v[3]} + for v in [shlex.split(g) for g in gcloud_record.rrdatas]]} + + def _data_for_TXT(self, gcloud_record): + if len(gcloud_record.rrdatas) > 1: + return { + 'values': gcloud_record.rrdatas} + return { + 'value': gcloud_record.rrdatas[0]} diff --git a/requirements.txt b/requirements.txt index a7d1d94..80fbe1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ dnspython==1.15.0 docutils==0.14 dyn==1.8.0 futures==3.1.1 +google-cloud==0.27.0 incf.countryutils==1.0 ipaddress==1.0.18 jmespath==0.9.3 diff --git a/script/test b/script/test index 3ee6e9c..41edfd8 100755 --- a/script/test +++ b/script/test @@ -24,5 +24,6 @@ export DNSIMPLE_TOKEN= export DYN_CUSTOMER= export DYN_PASSWORD= export DYN_USERNAME= +export GOOGLE_APPLICATION_CREDENTIALS= nosetests "$@" diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py new file mode 100644 index 0000000..568ac4f --- /dev/null +++ b/tests/test_octodns_provider_googlecloud.py @@ -0,0 +1,421 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from octodns.record import Create, Delete, Update, Record +from octodns.provider.googlecloud import GoogleCloudProvider, \ + _GoogleCloudRecordSetMaker + +from octodns.zone import Zone +from octodns.provider.base import Plan + +from unittest import TestCase +from mock import Mock, patch, PropertyMock + +zone = Zone(name='unit.tests.', sub_zones=[]) +octo_records = [] +octo_records.append(Record.new(zone, '', { + 'ttl': 0, + 'type': 'A', + 'values': ['1.2.3.4', '10.10.10.10']})) +octo_records.append(Record.new(zone, 'a', { + 'ttl': 1, + 'type': 'A', + 'values': ['1.2.3.4', '1.1.1.1']})) +octo_records.append(Record.new(zone, 'aa', { + 'ttl': 9001, + 'type': 'A', + 'values': ['1.2.4.3']})) +octo_records.append(Record.new(zone, 'aaa', { + 'ttl': 2, + 'type': 'A', + 'values': ['1.1.1.3']})) +octo_records.append(Record.new(zone, 'cname', { + 'ttl': 3, + 'type': 'CNAME', + 'value': 'a.unit.tests.'})) +octo_records.append(Record.new(zone, 'mx1', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }, { + 'priority': 20, + 'value': 'mx2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, 'mx2', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '', { + 'ttl': 4, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.']})) +octo_records.append(Record.new(zone, 'foo', { + 'ttl': 5, + 'type': 'NS', + 'value': 'ns1.unit.tests.'})) +octo_records.append(Record.new(zone, '_srv._tcp', { + 'ttl': 6, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 12, + 'weight': 30, + 'port': 30, + 'target': 'foo-2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '_srv2._tcp', { + 'ttl': 7, + 'type': 'SRV', + 'values': [{ + 'priority': 12, + 'weight': 17, + 'port': 1, + 'target': 'srvfoo.unit.tests.', + }]})) +octo_records.append(Record.new(zone, 'txt1', { + 'ttl': 8, + 'type': 'TXT', + 'value': 'txt singleton test'})) +octo_records.append(Record.new(zone, 'txt2', { + 'ttl': 9, + 'type': 'TXT', + 'values': ['txt multiple test', 'txt multiple test 2']})) +octo_records.append(Record.new(zone, 'naptr', { + 'ttl': 9, + 'type': 'NAPTR', + 'values': [{ + 'order': 100, + 'preference': 10, + 'flags': 'S', + 'service': 'SIP+D2U', + 'regexp': "!^.*$!sip:customer-service@unit.tests!", + 'replacement': '_sip._udp.unit.tests.' + }]})) +octo_records.append(Record.new(zone, 'caa', { + 'ttl': 9, + 'type': 'CAA', + 'value': { + 'flags': 0, + 'tag': 'issue', + 'value': 'ca.unit.tests', + }})) +for record in octo_records: + zone.add_record(record) + +# This is the format which the google API likes. +resource_record_sets = [ + ('unit.tests.', u'A', 0, [u'1.2.3.4', u'10.10.10.10']), + (u'a.unit.tests.', u'A', 1, [u'1.1.1.1', u'1.2.3.4']), + (u'aa.unit.tests.', u'A', 9001, [u'1.2.4.3']), + (u'aaa.unit.tests.', u'A', 2, [u'1.1.1.3']), + (u'cname.unit.tests.', u'CNAME', 3, [u'a.unit.tests.']), + (u'mx1.unit.tests.', u'MX', 3, + [u'10 mx1.unit.tests.', u'20 mx2.unit.tests.']), + (u'mx2.unit.tests.', u'MX', 3, [u'10 mx1.unit.tests.']), + ('unit.tests.', u'NS', 4, [u'ns1.unit.tests.', u'ns2.unit.tests.']), + (u'foo.unit.tests.', u'NS', 5, [u'ns1.unit.tests.']), + (u'_srv._tcp.unit.tests.', u'SRV', 6, + [u'10 20 30 foo-1.unit.tests.', u'12 30 30 foo-2.unit.tests.']), + (u'_srv2._tcp.unit.tests.', u'SRV', 7, [u'12 17 1 srvfoo.unit.tests.']), + (u'txt1.unit.tests.', u'TXT', 8, [u'txt singleton test']), + (u'txt2.unit.tests.', u'TXT', 9, + [u'txt multiple test', u'txt multiple test 2']), + (u'naptr.unit.tests.', u'NAPTR', 9, [ + u'100 10 "S" "SIP+D2U" "!^.*$!sip:customer-service@unit.tests!"' + u' _sip._udp.unit.tests.']), + (u'caa.unit.tests.', u'CAA', 9, [u'0 issue ca.unit.tests']) +] + + +class DummyResourceRecordSet: + def __init__(self, record_name, record_type, ttl, rrdatas): + self.name = record_name + self.record_type = record_type + self.ttl = ttl + self.rrdatas = rrdatas + + def __eq__(self, other): + try: + return self.name == other.name \ + and self.record_type == other.record_type \ + and self.ttl == other.ttl \ + and sorted(self.rrdatas) == sorted(other.rrdatas) + except: + return False + + def __repr__(self): + return "{} {} {} {!s}"\ + .format(self.name, self.record_type, self.ttl, self.rrdatas) + + def __hash__(self): + return hash(repr(self)) + + +class DummyGoogleCloudZone: + def __init__(self, dns_name): + self.dns_name = dns_name + + def resource_record_set(self, *args): + return DummyResourceRecordSet(*args) + + def list_resource_record_sets(self, *args): + pass + + +class DummyIterator: + """Returns a mock DummyIterator object to use in testing. + This is because API calls for google cloud DNS, if paged, contains a + "next_page_token", which can be used to grab a subsequent + iterator with more results. + + :type return: DummyIterator + """ + def __init__(self, list_of_stuff, page_token=None): + self.iterable = iter(list_of_stuff) + self.next_page_token = page_token + + def __iter__(self): + return self + + def next(self): + return self.iterable.next() + + +class TestGoogleCloudRecordSetMaker(TestCase): + def test_get_record_set(self): + mz = DummyGoogleCloudZone('unit.tests.') + record_sets = [] + for record in octo_records: + mm = _GoogleCloudRecordSetMaker(mz, record) + record_sets.append(mm.get_record_set()) + + self.assertEqual( + len(octo_records), + len(record_sets)) + + +class TestGoogleCloudProvider(TestCase): + @patch('octodns.provider.googlecloud.dns') + def _get_provider(*args): + '''Returns a mock GoogleCloudProvider object to use in testing. + + :type return: GoogleCloudProvider + ''' + return GoogleCloudProvider(id=1, project="mock") + + @patch('octodns.provider.googlecloud.time.sleep') + @patch('octodns.provider.googlecloud.dns') + def test__apply(self, *_): + class DummyDesired: + def __init__(self, name, changes): + self.name = name + self.changes = changes + + apply_z = Zone("unit.tests.", []) + create_r = Record.new(apply_z, '', { + 'ttl': 0, + 'type': 'A', + 'values': ['1.2.3.4', '10.10.10.10']}) + delete_r = Record.new(apply_z, 'a', { + 'ttl': 1, + 'type': 'A', + 'values': ['1.2.3.4', '1.1.1.1']}) + update_existing_r = Record.new(apply_z, 'aa', { + 'ttl': 9001, + 'type': 'A', + 'values': ['1.2.4.3']}) + update_new_r = Record.new(apply_z, 'aa', { + 'ttl': 666, + 'type': 'A', + 'values': ['1.4.3.2']}) + + gcloud_zone_mock = DummyGoogleCloudZone("unit.tests.") + status_mock = Mock() + return_values_for_status = iter( + ['', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + '', '', '', 'done']) + type(status_mock).status = PropertyMock( + side_effect=return_values_for_status.next) + gcloud_zone_mock.changes = Mock(return_value=status_mock) + + provider = self._get_provider() + provider.gcloud_client = Mock() + provider._get_gcloud_zone = Mock( + return_value=gcloud_zone_mock) + desired = Mock() + desired.name = Mock(return_value="unit.tests.") + changes = [] + changes.append(Create(create_r)) + changes.append(Delete(delete_r)) + changes.append(Update(existing=update_existing_r, new=update_new_r)) + + provider.apply(Plan( + existing=[update_existing_r, delete_r], + desired=desired, + changes=changes + )) + + calls_mock = gcloud_zone_mock.changes.return_value + mocked_calls = [] + for mock_call in calls_mock.add_record_set.mock_calls: + mocked_calls.append(mock_call[1][0]) + + self.assertEqual(mocked_calls, [ + DummyResourceRecordSet( + 'unit.tests.', 'A', 0, ['1.2.3.4', '10.10.10.10']), + DummyResourceRecordSet( + 'aa.unit.tests.', 'A', 666, ['1.4.3.2']) + ]) + + mocked_calls2 = [] + for mock_call in calls_mock.delete_record_set.mock_calls: + mocked_calls2.append(mock_call[1][0]) + + self.assertEqual(mocked_calls2, [ + DummyResourceRecordSet( + 'a.unit.tests.', 'A', 1, ['1.2.3.4', '1.1.1.1']), + DummyResourceRecordSet( + 'aa.unit.tests.', 'A', 9001, ['1.2.4.3']) + ]) + + unsupported_change = Mock() + unsupported_change.__len__ = Mock(return_value=1) + mock_plan = Mock() + type(mock_plan).desired = PropertyMock(return_value=DummyDesired( + "dummy name", [])) + type(mock_plan).changes = [unsupported_change] + + with self.assertRaises(RuntimeError): + provider.apply(mock_plan) + + def test__record_to_record_set(self): + provider = self._get_provider() + gcloud_zone = DummyGoogleCloudZone('unit.tests.') + for record in octo_records: + self.assertIsNotNone(provider._record_to_record_set( + gcloud_zone, record)) + + def test__get_gcloud_client(self): + provider = self._get_provider() + + self.assertIsInstance(provider, GoogleCloudProvider) + + @patch('octodns.provider.googlecloud.dns') + def test_populate(self, _): + def _get_mock_zones(page_token=None): + if not page_token: + return DummyIterator([ + DummyGoogleCloudZone('example.com.'), + DummyGoogleCloudZone('example2.com.'), + ], page_token="DUMMY_PAGE_TOKEN") + + return DummyIterator([ + google_cloud_zone + ]) + + def _get_mock_record_sets(page_token=None): + if not page_token: + return DummyIterator( + [DummyResourceRecordSet(*v) for v in + resource_record_sets[:5]], page_token="DUMMY_PAGE_TOKEN") + return DummyIterator( + [DummyResourceRecordSet(*v) for v in resource_record_sets[5:]]) + + google_cloud_zone = DummyGoogleCloudZone('unit.tests.') + + provider = self._get_provider() + provider.gcloud_client.list_zones = Mock(side_effect=_get_mock_zones) + google_cloud_zone.list_resource_record_sets = Mock( + side_effect=_get_mock_record_sets) + + self.assertEqual(provider._get_gcloud_zone("unit.tests.").dns_name, + "unit.tests.") + + test_zone = Zone('unit.tests.', []) + provider.populate(test_zone) + + # test_zone gets fed the same records as zone does, except it's in + # the format returned by google API, so after populate they should look + # excactly the same. + self.assertEqual(test_zone.records, zone.records) + + test_zone2 = Zone('nonexistant.zone.', []) + provider.populate(test_zone2, False, False) + + self.assertEqual(len(test_zone2.records), 0, + msg="Zone should not get records from wrong domain") + + provider.SUPPORTS = set() + test_zone3 = Zone('unit.tests.', []) + provider.populate(test_zone3) + self.assertEqual(len(test_zone3.records), 0) + + @patch('octodns.provider.googlecloud.dns') + def test_populate_corner_cases(self, _): + provider = self._get_provider() + test_zone = Zone('unit.tests.', []) + not_same_fqdn = DummyResourceRecordSet( + 'unit.tests.gr', u'A', 0, [u'1.2.3.4']), + + provider._get_gcloud_records = Mock( + side_effect=[not_same_fqdn]) + provider._get_gcloud_zone = Mock(return_value=DummyGoogleCloudZone( + dns_name="unit.tests.")) + + provider.populate(test_zone) + + self.assertEqual(len(test_zone.records), 1) + + self.assertEqual(test_zone.records.pop().fqdn, + u'unit.tests.gr.unit.tests.') + + def test__get_gcloud_zone(self): + provider = self._get_provider() + + provider.gcloud_client = Mock() + provider.gcloud_client.list_zones = Mock( + return_value=DummyIterator([])) + + self.assertIsNone(provider._get_gcloud_zone("nonexistant.xone"), + msg="Check that nonexistant zones return None when" + "there's no create=True flag") + + def test__create_zone(self): + provider = self._get_provider() + + provider.gcloud_client = Mock() + provider.gcloud_client.list_zones = Mock( + return_value=DummyIterator([])) + + mock_zone = provider._get_gcloud_zone( + 'nonexistant.zone.mock', create=True) + + mock_zone.create.assert_called() + provider.gcloud_client.zone.assert_called() + provider.gcloud_client.zone.assert_called_once_with( + dns_name=u'nonexistant.zone.mock', name=u'nonexistant-zone-moc') + + def test__create_zone_with_numbers_in_name(self): + provider = self._get_provider() + + provider.gcloud_client = Mock() + provider.gcloud_client.list_zones = Mock( + return_value=DummyIterator([])) + + provider._get_gcloud_zone( + '111.', create=True) + provider.gcloud_client.zone.assert_called_once_with( + dns_name=u'111.', name=u'a111') From f082d5798f10a9f68b90acccc408ef8617b6e5fb Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Tue, 3 Oct 2017 09:12:02 +0200 Subject: [PATCH 68/84] Revert "Add .idea (intelliJ) files to gitignore." This reverts commit 5a3d844559f1d1514a171f1fa38d80a27a096ce9. --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1bca124..c45a684 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,3 @@ output/ tmp/ build/ config/ -.idea/ \ No newline at end of file From 2a3690e8778a4bbaf5d84891a4cea23f38b68e61 Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Tue, 3 Oct 2017 11:12:25 +0200 Subject: [PATCH 69/84] Add auth config opts to googlecloud provider Also make _data_for_SPF and _data_for_TXT the same method. --- octodns/provider/googlecloud.py | 32 +++++++++++++++------- tests/test_octodns_provider_googlecloud.py | 13 ++++++++- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index dff11fc..8aa8c99 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -96,22 +96,34 @@ class GoogleCloudProvider(BaseProvider): # specified with the GOOGLE_APPLICATION_CREDENTIALS environment # variable. (https://console.cloud.google.com/apis/credentials) # - # The project to work on (not required) + # The project to work on (not required) # project: foobar + # + # The File with the google credentials (not required). If used, the + # "project" parameter needs to be set, else it will fall back to the + # "default credentials" + # credentials_file: ~/google_cloud_credentials_file.json + # """ SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) SUPPORTS_GEO = False - def __init__(self, id, project=None, *args, **kwargs): + def __init__(self, id, project=None, credentials_file=None, + *args, **kwargs): + + if credentials_file: + self.gcloud_client = dns.Client.from_service_account_json( + credentials_file, project=project) + else: + self.gcloud_client = dns.Client(project=project) # Logger self.log = getLogger('GoogleCloudProvider[{}]'.format(id)) self.id = id super(GoogleCloudProvider, self).__init__(id, *args, **kwargs) - self.gcloud_client = dns.Client(project=project) def _apply(self, plan): """Required function of manager.py to actually apply a record change. @@ -345,7 +357,12 @@ class GoogleCloudProvider(BaseProvider): _data_for_PTR = _data_for_CNAME - _data_for_SPF = _data_for_A + def _data_for_SPF(self, gcloud_record): + if len(gcloud_record.rrdatas) > 1: + return { + 'values': gcloud_record.rrdatas} + return { + 'value': gcloud_record.rrdatas[0]} def _data_for_SRV(self, gcloud_record): return {'values': [{ @@ -355,9 +372,4 @@ class GoogleCloudProvider(BaseProvider): 'target': v[3]} for v in [shlex.split(g) for g in gcloud_record.rrdatas]]} - def _data_for_TXT(self, gcloud_record): - if len(gcloud_record.rrdatas) > 1: - return { - 'values': gcloud_record.rrdatas} - return { - 'value': gcloud_record.rrdatas[0]} + _data_for_TXT = _data_for_SPF diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index 568ac4f..c7b93e3 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -10,7 +10,7 @@ from octodns.provider.googlecloud import GoogleCloudProvider, \ _GoogleCloudRecordSetMaker from octodns.zone import Zone -from octodns.provider.base import Plan +from octodns.provider.base import Plan, BaseProvider from unittest import TestCase from mock import Mock, patch, PropertyMock @@ -216,6 +216,17 @@ class TestGoogleCloudProvider(TestCase): ''' return GoogleCloudProvider(id=1, project="mock") + @patch('octodns.provider.googlecloud.time.sleep') + @patch('octodns.provider.googlecloud.dns') + def test___init__(self, *_): + self.assertIsInstance(GoogleCloudProvider(id=1, + credentials_file="test", + project="unit test"), + BaseProvider) + + self.assertIsInstance(GoogleCloudProvider(id=1), + BaseProvider) + @patch('octodns.provider.googlecloud.time.sleep') @patch('octodns.provider.googlecloud.dns') def test__apply(self, *_): From 8230700ad17ebe9f8fcaf80212bec9a424d5f1ec Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Tue, 3 Oct 2017 13:25:36 +0200 Subject: [PATCH 70/84] Consolidate googlecloud provider to single class remove _GoogleCloudRecordSetMaker into the GoogleCloudProvider, and consolidate methods. --- octodns/provider/googlecloud.py | 144 ++++++++------------- tests/test_octodns_provider_googlecloud.py | 38 +++--- 2 files changed, 72 insertions(+), 110 deletions(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index 8aa8c99..0f20587 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -16,76 +16,6 @@ from .base import BaseProvider from ..record import Record -class _GoogleCloudRecordSetMaker(object): - """Wrapper to make google cloud client resource record sets from OctoDNS - Records. - - googlecloud.py: - class: octodns.provider.googlecloud._GoogleCloudRecordSetMaker - An _GoogleCloudRecordSetMaker creates google cloued client resource - records which can be used to update the Google Cloud DNS zones. - """ - - def __init__(self, gcloud_zone, record): - self.gcloud_zone = gcloud_zone - self.record = record - - self._record_set_func = getattr( - self, '_record_set_from_{}'.format(record._type)) - - def get_record_set(self): - return self._record_set_func(self.record) - - def _record_set_from_A(self, record): - return self.gcloud_zone.resource_record_set( - record.fqdn, record._type, record.ttl, record.values) - - _record_set_from_AAAA = _record_set_from_A - - def _record_set_from_CAA(self, record): - return self.gcloud_zone.resource_record_set( - record.fqdn, record._type, record.ttl, [ - '{flags} {tag} {value}'.format(**record.data['value'])]) - - def _record_set_from_CNAME(self, record): - return self.gcloud_zone.resource_record_set( - record.fqdn, record._type, record.ttl, [record.value]) - - def _record_set_from_MX(self, record): - return self.gcloud_zone.resource_record_set( - record.fqdn, record._type, record.ttl, [ - '{preference} {exchange}'.format(**v.data) - for v in record.values]) - - def _record_set_from_NAPTR(self, record): - return self.gcloud_zone.resource_record_set( - record.fqdn, record._type, record.ttl, [ - '{order} {preference} "{flags}" "{service}" ' - '"{regexp}" {replacement}' - .format(**v.data) for v in record.values]) - - _record_set_from_NS = _record_set_from_A - - _record_set_from_PTR = _record_set_from_CNAME - - _record_set_from_SPF = _record_set_from_A - - def _record_set_from_SRV(self, record): - return self.gcloud_zone.resource_record_set( - record.fqdn, record._type, record.ttl, [ - '{priority} {weight} {port} {target}' - .format(**v.data) for v in record.values]) - - def _record_set_from_TXT(self, record): - if 'values' in record.data: - val = record.data['values'] - else: - val = [record.data['value']] - - return self.gcloud_zone.resource_record_set( - record.fqdn, record._type, record.ttl, val) - - class GoogleCloudProvider(BaseProvider): """ Google Cloud DNS provider @@ -146,17 +76,20 @@ class GoogleCloudProvider(BaseProvider): for change in changes: class_name = change.__class__.__name__ + _rrset_func = getattr( + self, '_rrset_for_{}'.format(change.record._type)) + if class_name in 'Create': gcloud_changes.add_record_set( - self._record_to_record_set(gcloud_zone, change.record)) + _rrset_func(gcloud_zone, change.record)) elif class_name == 'Delete': gcloud_changes.delete_record_set( - self._record_to_record_set(gcloud_zone, change.record)) + _rrset_func(gcloud_zone, change.record)) elif class_name == 'Update': gcloud_changes.delete_record_set( - self._record_to_record_set(gcloud_zone, change.existing)) + _rrset_func(gcloud_zone, change.existing)) gcloud_changes.add_record_set( - self._record_to_record_set(gcloud_zone, change.new)) + _rrset_func(gcloud_zone, change.new)) else: raise RuntimeError('Change type "{}" for change "{!s}" ' 'is none of "Create", "Delete" or "Update' @@ -260,20 +193,6 @@ class GoogleCloudProvider(BaseProvider): if create: return self._create_gcloud_zone(dns_name) - @staticmethod - def _record_to_record_set(gcloud_zone, record): - """create google.cloud.dns.ResourceRecordSet from ocdodns.Record - - :param record: a record object - :type record: ocdodns.Record - :param gcloud_zone: a google gcloud zone - :type gcloud_zone: google.cloud.dns.ManagedZone - :type return: google.cloud.dns.ResourceRecordSet - """ - grm = _GoogleCloudRecordSetMaker(gcloud_zone, record) - - return grm.get_record_set() - def populate(self, zone, target=False, lenient=False): """Required function of manager.py to collect records from zone. @@ -373,3 +292,52 @@ class GoogleCloudProvider(BaseProvider): for v in [shlex.split(g) for g in gcloud_record.rrdatas]]} _data_for_TXT = _data_for_SPF + + def _rrset_for_A(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, record.values) + + _rrset_for_AAAA = _rrset_for_A + + def _rrset_for_CAA(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{flags} {tag} {value}'.format(**record.data['value'])]) + + def _rrset_for_CNAME(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [record.value]) + + def _rrset_for_MX(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{preference} {exchange}'.format(**v.data) + for v in record.values]) + + def _rrset_for_NAPTR(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{order} {preference} "{flags}" "{service}" ' + '"{regexp}" {replacement}' + .format(**v.data) for v in record.values]) + + _rrset_for_NS = _rrset_for_A + + _rrset_for_PTR = _rrset_for_CNAME + + def _rrset_for_SPF(self, gcloud_zone, record): + if 'values' in record.data: + val = record.data['values'] + else: + val = [record.data['value']] + + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, val) + + def _rrset_for_SRV(self, gcloud_zone, record): + return gcloud_zone.resource_record_set( + record.fqdn, record._type, record.ttl, [ + '{priority} {weight} {port} {target}' + .format(**v.data) for v in record.values]) + + _rrset_for_TXT = _rrset_for_SPF diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index c7b93e3..76eedba 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -6,8 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from octodns.record import Create, Delete, Update, Record -from octodns.provider.googlecloud import GoogleCloudProvider, \ - _GoogleCloudRecordSetMaker +from octodns.provider.googlecloud import GoogleCloudProvider from octodns.zone import Zone from octodns.provider.base import Plan, BaseProvider @@ -194,19 +193,6 @@ class DummyIterator: return self.iterable.next() -class TestGoogleCloudRecordSetMaker(TestCase): - def test_get_record_set(self): - mz = DummyGoogleCloudZone('unit.tests.') - record_sets = [] - for record in octo_records: - mm = _GoogleCloudRecordSetMaker(mz, record) - record_sets.append(mm.get_record_set()) - - self.assertEqual( - len(octo_records), - len(record_sets)) - - class TestGoogleCloudProvider(TestCase): @patch('octodns.provider.googlecloud.dns') def _get_provider(*args): @@ -304,6 +290,10 @@ class TestGoogleCloudProvider(TestCase): unsupported_change = Mock() unsupported_change.__len__ = Mock(return_value=1) + type_mock = Mock() + type_mock._type = "A" + unsupported_change.record = type_mock + mock_plan = Mock() type(mock_plan).desired = PropertyMock(return_value=DummyDesired( "dummy name", [])) @@ -312,13 +302,6 @@ class TestGoogleCloudProvider(TestCase): with self.assertRaises(RuntimeError): provider.apply(mock_plan) - def test__record_to_record_set(self): - provider = self._get_provider() - gcloud_zone = DummyGoogleCloudZone('unit.tests.') - for record in octo_records: - self.assertIsNotNone(provider._record_to_record_set( - gcloud_zone, record)) - def test__get_gcloud_client(self): provider = self._get_provider() @@ -404,6 +387,17 @@ class TestGoogleCloudProvider(TestCase): msg="Check that nonexistant zones return None when" "there's no create=True flag") + def test__get_rrsets(self): + provider = self._get_provider() + dummy_gcloud_zone = DummyGoogleCloudZone("unit.tests") + for octo_record in octo_records: + _rrset_func = getattr( + provider, '_rrset_for_{}'.format(octo_record._type)) + self.assertEqual( + _rrset_func(dummy_gcloud_zone, octo_record).record_type, + octo_record._type + ) + def test__create_zone(self): provider = self._get_provider() From aabab630030ba7ad17ff705b4e89d082c9b0f413 Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Sat, 7 Oct 2017 16:13:11 +0200 Subject: [PATCH 71/84] Refactor GoogleCloudProvider * in _rrset_for_X functions, use values instead of data attribute. * Small typo fixes and removals of redundant steps etc. * Unset GOOGLE_APPLICATION_CREDENTIALS in coverage script. --- octodns/provider/googlecloud.py | 34 ++++++++++------------ script/coverage | 1 + tests/test_octodns_provider_googlecloud.py | 12 ++++++-- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index 0f20587..103c3f5 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -79,7 +79,7 @@ class GoogleCloudProvider(BaseProvider): _rrset_func = getattr( self, '_rrset_for_{}'.format(change.record._type)) - if class_name in 'Create': + if class_name == 'Create': gcloud_changes.add_record_set( _rrset_func(gcloud_zone, change.record)) elif class_name == 'Delete': @@ -212,17 +212,16 @@ class GoogleCloudProvider(BaseProvider): gcloud_zone = self._get_gcloud_zone(zone.name) - _records = set() if gcloud_zone: for gcloud_record in self._get_gcloud_records(gcloud_zone): - if gcloud_record.record_type.upper() in self.SUPPORTS: - _records.add(gcloud_record) - for gcloud_record in _records: + if gcloud_record.record_type.upper() not in self.SUPPORTS: + continue + record_name = gcloud_record.name if record_name.endswith(zone.name): # google cloud always return fqdn. Make relative record # here. "root" records will then get the '' record_name, - # which is also the way dyn likes it. + # which is also the way octodns likes it. record_name = record_name[:-(len(zone.name) + 1)] typ = gcloud_record.record_type.upper() data = getattr(self, '_data_for_{}'.format(typ)) @@ -302,7 +301,8 @@ class GoogleCloudProvider(BaseProvider): def _rrset_for_CAA(self, gcloud_zone, record): return gcloud_zone.resource_record_set( record.fqdn, record._type, record.ttl, [ - '{flags} {tag} {value}'.format(**record.data['value'])]) + '{} {} {}'.format(v.flags, v.tag, v.value) + for v in record.values]) def _rrset_for_CNAME(self, gcloud_zone, record): return gcloud_zone.resource_record_set( @@ -311,33 +311,29 @@ class GoogleCloudProvider(BaseProvider): def _rrset_for_MX(self, gcloud_zone, record): return gcloud_zone.resource_record_set( record.fqdn, record._type, record.ttl, [ - '{preference} {exchange}'.format(**v.data) + '{} {}'.format(v.preference, v.exchange) for v in record.values]) def _rrset_for_NAPTR(self, gcloud_zone, record): return gcloud_zone.resource_record_set( record.fqdn, record._type, record.ttl, [ - '{order} {preference} "{flags}" "{service}" ' - '"{regexp}" {replacement}' - .format(**v.data) for v in record.values]) + '{} {} "{}" "{}" "{}" {}'.format( + v.order, v.preference, v.flags, v.service, + v.regexp, v.replacement) for v in record.values]) _rrset_for_NS = _rrset_for_A _rrset_for_PTR = _rrset_for_CNAME def _rrset_for_SPF(self, gcloud_zone, record): - if 'values' in record.data: - val = record.data['values'] - else: - val = [record.data['value']] - return gcloud_zone.resource_record_set( - record.fqdn, record._type, record.ttl, val) + record.fqdn, record._type, record.ttl, record.values) def _rrset_for_SRV(self, gcloud_zone, record): return gcloud_zone.resource_record_set( record.fqdn, record._type, record.ttl, [ - '{priority} {weight} {port} {target}' - .format(**v.data) for v in record.values]) + '{} {} {} {}' + .format(v.priority, v.weight, v.port, v.target) + for v in record.values]) _rrset_for_TXT = _rrset_for_SPF diff --git a/script/coverage b/script/coverage index ca5d693..228a772 100755 --- a/script/coverage +++ b/script/coverage @@ -24,6 +24,7 @@ export DNSIMPLE_TOKEN= export DYN_CUSTOMER= export DYN_PASSWORD= export DYN_USERNAME= +export GOOGLE_APPLICATION_CREDENTIALS= coverage run --branch --source=octodns `which nosetests` --with-xunit "$@" coverage html diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index 76eedba..b498d4c 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -313,8 +313,11 @@ class TestGoogleCloudProvider(TestCase): if not page_token: return DummyIterator([ DummyGoogleCloudZone('example.com.'), + ], page_token="MOCK_PAGE_TOKEN") + elif page_token == "MOCK_PAGE_TOKEN": + return DummyIterator([ DummyGoogleCloudZone('example2.com.'), - ], page_token="DUMMY_PAGE_TOKEN") + ], page_token="MOCK_PAGE_TOKEN2") return DummyIterator([ google_cloud_zone @@ -324,7 +327,12 @@ class TestGoogleCloudProvider(TestCase): if not page_token: return DummyIterator( [DummyResourceRecordSet(*v) for v in - resource_record_sets[:5]], page_token="DUMMY_PAGE_TOKEN") + resource_record_sets[:3]], page_token="MOCK_PAGE_TOKEN") + elif page_token == "MOCK_PAGE_TOKEN": + + return DummyIterator( + [DummyResourceRecordSet(*v) for v in + resource_record_sets[3:5]], page_token="MOCK_PAGE_TOKEN2") return DummyIterator( [DummyResourceRecordSet(*v) for v in resource_record_sets[5:]]) From 4b878b844660b85f9cd1a9cc364850060cb3ec3d Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Sat, 7 Oct 2017 19:31:23 +0200 Subject: [PATCH 72/84] Cache encountered zones in GoogleCloudProvider Cache googleclouds zones so that populate dont have to list all each time called. --- octodns/provider/googlecloud.py | 76 +++++++++++----------- tests/test_octodns_provider_googlecloud.py | 46 ++++++++----- 2 files changed, 68 insertions(+), 54 deletions(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index 103c3f5..fb85744 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -53,6 +53,8 @@ class GoogleCloudProvider(BaseProvider): self.log = getLogger('GoogleCloudProvider[{}]'.format(id)) self.id = id + self._gcloud_zones = {} + super(GoogleCloudProvider, self).__init__(id, *args, **kwargs) def _apply(self, plan): @@ -70,7 +72,10 @@ class GoogleCloudProvider(BaseProvider): len(changes)) # Get gcloud zone, or create one if none existed before. - gcloud_zone = self._get_gcloud_zone(desired.name, create=True) + if desired.name not in self.gcloud_zones: + gcloud_zone = self._create_gcloud_zone(desired.name) + else: + gcloud_zone = self.gcloud_zones.get(desired.name) gcloud_changes = gcloud_zone.changes() @@ -117,13 +122,19 @@ class GoogleCloudProvider(BaseProvider): # and only contain lowercase letters, digits or dashes zone_name = re.sub("[^a-z0-9-]", "", dns_name[:-1].replace('.', "-")) - # make sure that the end result did not end up wo leading letter - if re.match('[^a-z]', zone_name[0]): - # I cannot think of a situation where a zone name derived from - # a domain name would'nt start with leading letter and thereby - # violate the constraint, however if such a situation is - # encountered, add a leading "a" here. - zone_name = "a%s" % zone_name + + # Check if there is another zone in google cloud which has the same + # name as the new one + while zone_name in [z.name for z in self.gcloud_zones.values()]: + # If there is a zone in google cloud alredy, then try suffixing the + # name with a -i where i is a number which keeps increasing until + # a free name has been reached. + m = re.match("^(.+)-([0-9]+$)", zone_name) + if m: + i = int(m.group(2)) + 1 + zone_name = "{}-{!s}".format(m.group(1), i) + else: + zone_name += "-2" gcloud_zone = self.gcloud_client.zone( name=zone_name, @@ -131,6 +142,9 @@ class GoogleCloudProvider(BaseProvider): ) gcloud_zone.create(client=self.gcloud_client) + # add this new zone to the list of zones. + self._gcloud_zones[gcloud_zone.dns_name] = gcloud_zone + self.log.info("Created zone %s. Fqdn %s." % (zone_name, dns_name)) @@ -159,39 +173,25 @@ class GoogleCloudProvider(BaseProvider): # yield from is in python 3 only. yield gcloud_record - def _get_gcloud_zone(self, dns_name, page_token=None, create=False): - """Return the ManagedZone which has has the matching dns_name, or - None if no such zone exist, unless create=True, then create a new - one and return it. + def _get_cloud_zones(self, page_token=None): + """Load all ManagedZones into the self._gcloud_zones dict which is + mapped with the dns_name as key. - :param dns_name: fqdn of dns name for zone to get. - :type dns_name: str - :param page_token: page token for the page to get - :type page_token: str - :param create: if true, create ManagedZone if it does not exist - already - - :type return: new google.cloud.dns.ManagedZone + :return: void """ - # Find the google name for the incoming zone + gcloud_zones = self.gcloud_client.list_zones(page_token=page_token) for gcloud_zone in gcloud_zones: - if gcloud_zone.dns_name == dns_name: - return gcloud_zone - else: - # Zone not found. Check if there are more results which could be - # retrieved by checking "next_page_token". - if gcloud_zones.next_page_token: - return self._get_gcloud_zone(dns_name, - gcloud_zones.next_page_token) - else: - # Nothing found, either return None or else create zone and - # return that one (if create=True) - self.log.debug('_get_gcloud_zone: zone name=%s, ' - 'was not found by %s.', - dns_name, self.gcloud_client) - if create: - return self._create_gcloud_zone(dns_name) + self._gcloud_zones[gcloud_zone.dns_name] = gcloud_zone + + if gcloud_zones.next_page_token: + self._get_cloud_zones(gcloud_zones.next_page_token) + + @property + def gcloud_zones(self): + if not self._gcloud_zones: + self._get_cloud_zones() + return self._gcloud_zones def populate(self, zone, target=False, lenient=False): """Required function of manager.py to collect records from zone. @@ -210,7 +210,7 @@ class GoogleCloudProvider(BaseProvider): target, lenient) before = len(zone.records) - gcloud_zone = self._get_gcloud_zone(zone.name) + gcloud_zone = self.gcloud_zones.get(zone.name) if gcloud_zone: for gcloud_record in self._get_gcloud_records(gcloud_zone): diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index b498d4c..fa667cc 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -164,8 +164,9 @@ class DummyResourceRecordSet: class DummyGoogleCloudZone: - def __init__(self, dns_name): + def __init__(self, dns_name, name=""): self.dns_name = dns_name + self.name = name def resource_record_set(self, *args): return DummyResourceRecordSet(*args) @@ -173,6 +174,9 @@ class DummyGoogleCloudZone: def list_resource_record_sets(self, *args): pass + def create(self, *args, **kwargs): + pass + class DummyIterator: """Returns a mock DummyIterator object to use in testing. @@ -239,7 +243,7 @@ class TestGoogleCloudProvider(TestCase): 'type': 'A', 'values': ['1.4.3.2']}) - gcloud_zone_mock = DummyGoogleCloudZone("unit.tests.") + gcloud_zone_mock = DummyGoogleCloudZone("unit.tests.", "unit-tests") status_mock = Mock() return_values_for_status = iter( ['', '', '', '', '', '', '', '', '', '', '', '', '', '', '', @@ -250,10 +254,9 @@ class TestGoogleCloudProvider(TestCase): provider = self._get_provider() provider.gcloud_client = Mock() - provider._get_gcloud_zone = Mock( - return_value=gcloud_zone_mock) + provider._gcloud_zones = {"unit.tests.": gcloud_zone_mock} desired = Mock() - desired.name = Mock(return_value="unit.tests.") + desired.name = "unit.tests." changes = [] changes.append(Create(create_r)) changes.append(Delete(delete_r)) @@ -343,7 +346,7 @@ class TestGoogleCloudProvider(TestCase): google_cloud_zone.list_resource_record_sets = Mock( side_effect=_get_mock_record_sets) - self.assertEqual(provider._get_gcloud_zone("unit.tests.").dns_name, + self.assertEqual(provider.gcloud_zones.get("unit.tests.").dns_name, "unit.tests.") test_zone = Zone('unit.tests.', []) @@ -374,8 +377,8 @@ class TestGoogleCloudProvider(TestCase): provider._get_gcloud_records = Mock( side_effect=[not_same_fqdn]) - provider._get_gcloud_zone = Mock(return_value=DummyGoogleCloudZone( - dns_name="unit.tests.")) + provider._gcloud_zones = { + "unit.tests.": DummyGoogleCloudZone("unit.tests.", "unit-tests")} provider.populate(test_zone) @@ -391,7 +394,7 @@ class TestGoogleCloudProvider(TestCase): provider.gcloud_client.list_zones = Mock( return_value=DummyIterator([])) - self.assertIsNone(provider._get_gcloud_zone("nonexistant.xone"), + self.assertIsNone(provider.gcloud_zones.get("nonexistant.xone"), msg="Check that nonexistant zones return None when" "there's no create=True flag") @@ -413,22 +416,33 @@ class TestGoogleCloudProvider(TestCase): provider.gcloud_client.list_zones = Mock( return_value=DummyIterator([])) - mock_zone = provider._get_gcloud_zone( - 'nonexistant.zone.mock', create=True) + mock_zone = provider._create_gcloud_zone("nonexistant.zone.mock") mock_zone.create.assert_called() provider.gcloud_client.zone.assert_called() provider.gcloud_client.zone.assert_called_once_with( dns_name=u'nonexistant.zone.mock', name=u'nonexistant-zone-moc') - def test__create_zone_with_numbers_in_name(self): + def test__create_zone_with_duplicate_names(self): + + def _create_dummy_zone(name, dns_name): + return DummyGoogleCloudZone(name=name, dns_name=dns_name) + provider = self._get_provider() provider.gcloud_client = Mock() + provider.gcloud_client.zone = Mock(side_effect=_create_dummy_zone) provider.gcloud_client.list_zones = Mock( return_value=DummyIterator([])) - provider._get_gcloud_zone( - '111.', create=True) - provider.gcloud_client.zone.assert_called_once_with( - dns_name=u'111.', name=u'a111') + _gcloud_zones = { + 'unit-tests': DummyGoogleCloudZone("a.unit-tests.", "unit-tests") + } + + provider._gcloud_zones = _gcloud_zones + + test_zone_1 = provider._create_gcloud_zone("unit.tests.") + self.assertEqual(test_zone_1.name, "unit-tests-2") + + test_zone_2 = provider._create_gcloud_zone("unit.tests.") + self.assertEqual(test_zone_2.name, "unit-tests-3") From e9d90bda2bd11ec740c73162c5af268183ea76fe Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Sat, 7 Oct 2017 20:46:35 +0200 Subject: [PATCH 73/84] Add timeout logic to googlecloud provider --- octodns/provider/googlecloud.py | 21 +++++++++++++++------ tests/test_octodns_provider_googlecloud.py | 13 ++++++++++--- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index fb85744..e339c74 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -40,6 +40,8 @@ class GoogleCloudProvider(BaseProvider): 'NS', 'PTR', 'SPF', 'SRV', 'TXT')) SUPPORTS_GEO = False + CHANGE_LOOP_WAIT = 5 + def __init__(self, id, project=None, credentials_file=None, *args, **kwargs): @@ -101,13 +103,20 @@ class GoogleCloudProvider(BaseProvider): .format(class_name, change)) gcloud_changes.create() - i = 1 - while gcloud_changes.status != 'done': - self.log.debug("Waiting for changes to complete") - time.sleep(i) + + for i in range(120): gcloud_changes.reload() - if i < 30: - i += 2 + self.log.debug("Waiting for changes to complete") + # https://cloud.google.com/dns/api/v1/changes#resource + # status can be one of either "pending" or "done" + if gcloud_changes.status != 'pending': + break + self.log.debug("Waiting for changes to complete") + time.sleep(self.CHANGE_LOOP_WAIT) + + if gcloud_changes.status != 'done': + raise RuntimeError("Timeout reached after {} seconds".format( + i * self.CHANGE_LOOP_WAIT)) def _create_gcloud_zone(self, dns_name): """Creates a google cloud ManagedZone with dns_name, and zone named diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index fa667cc..c2e976c 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -206,7 +206,6 @@ class TestGoogleCloudProvider(TestCase): ''' return GoogleCloudProvider(id=1, project="mock") - @patch('octodns.provider.googlecloud.time.sleep') @patch('octodns.provider.googlecloud.dns') def test___init__(self, *_): self.assertIsInstance(GoogleCloudProvider(id=1, @@ -246,8 +245,7 @@ class TestGoogleCloudProvider(TestCase): gcloud_zone_mock = DummyGoogleCloudZone("unit.tests.", "unit-tests") status_mock = Mock() return_values_for_status = iter( - ['', '', '', '', '', '', '', '', '', '', '', '', '', '', '', - '', '', '', 'done']) + ["pending"] * 11 + ['done', 'done']) type(status_mock).status = PropertyMock( side_effect=return_values_for_status.next) gcloud_zone_mock.changes = Mock(return_value=status_mock) @@ -291,6 +289,15 @@ class TestGoogleCloudProvider(TestCase): 'aa.unit.tests.', 'A', 9001, ['1.2.4.3']) ]) + type(status_mock).status = "pending" + + with self.assertRaises(RuntimeError): + provider.apply(Plan( + existing=[update_existing_r, delete_r], + desired=desired, + changes=changes + )) + unsupported_change = Mock() unsupported_change.__len__ = Mock(return_value=1) type_mock = Mock() From ea1871a326d04243b6f632daa588e3bf15384e8f Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Sat, 7 Oct 2017 20:50:49 +0200 Subject: [PATCH 74/84] Add GoogleCloudProvider to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index afe92ac..a910b5b 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,7 @@ The above command pulled the existing data out of Route53 and placed the results | [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | +| [GoogleCloudProvider](/octodns/provider/googlecloud.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | All | No | | | [OVH](/octodns/provider/ovh.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | | | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | | From f50db5e02b54c8c54aaedee6411d9f94b495aec6 Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Mon, 9 Oct 2017 20:03:01 +0200 Subject: [PATCH 75/84] Use chunked_values in GoogleCloudProvider --- octodns/provider/googlecloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index e339c74..c544d63 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -336,7 +336,7 @@ class GoogleCloudProvider(BaseProvider): def _rrset_for_SPF(self, gcloud_zone, record): return gcloud_zone.resource_record_set( - record.fqdn, record._type, record.ttl, record.values) + record.fqdn, record._type, record.ttl, record.chunked_values) def _rrset_for_SRV(self, gcloud_zone, record): return gcloud_zone.resource_record_set( From a012e923f61547316d3e241d9e38b22a19ee1704 Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 10 Oct 2017 13:54:52 -0700 Subject: [PATCH 76/84] add ability to configure update/delete thresholds --- octodns/provider/base.py | 25 ++++++++++++++++++++----- tests/test_octodns_provider_base.py | 2 ++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index d2561ba..86175e7 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -21,10 +21,14 @@ class Plan(object): MAX_SAFE_DELETE_PCENT = .3 MIN_EXISTING_RECORDS = 10 - def __init__(self, existing, desired, changes): + def __init__(self, existing, desired, changes, + update_pcent_threshold=MAX_SAFE_UPDATE_PCENT, + delete_pcent_threshold=MAX_SAFE_DELETE_PCENT): self.existing = existing self.desired = desired self.changes = changes + self.update_pcent_threshold = update_pcent_threshold + self.delete_pcent_threshold = delete_pcent_threshold change_counts = { 'Create': 0, @@ -55,14 +59,19 @@ class Plan(object): update_pcent = self.change_counts['Update'] / existing_record_count delete_pcent = self.change_counts['Delete'] / existing_record_count - if update_pcent > self.MAX_SAFE_UPDATE_PCENT: + self.log.debug('raise_if_unsafe: update_pcent_threshold=%d, ' + 'delete_pcent_threshold=%d', + self.update_pcent_threshold, + self.delete_pcent_threshold) + + if update_pcent > self.update_pcent_threshold: raise UnsafePlan('Too many updates, {} is over {} percent' '({}/{})'.format( update_pcent, self.MAX_SAFE_UPDATE_PCENT * 100, self.change_counts['Update'], existing_record_count)) - if delete_pcent > self.MAX_SAFE_DELETE_PCENT: + if delete_pcent > self.delete_pcent_threshold: raise UnsafePlan('Too many deletes, {} is over {} percent' '({}/{})'.format( delete_pcent, @@ -79,11 +88,15 @@ class Plan(object): class BaseProvider(BaseSource): - def __init__(self, id, apply_disabled=False): + def __init__(self, id, apply_disabled=False, + update_pcent_threshold=Plan.MAX_SAFE_UPDATE_PCENT, + delete_pcent_threshold=Plan.MAX_SAFE_DELETE_PCENT): super(BaseProvider, self).__init__(id) self.log.debug('__init__: id=%s, apply_disabled=%s', id, apply_disabled) self.apply_disabled = apply_disabled + self.update_pcent_threshold = update_pcent_threshold + self.delete_pcent_threshold = delete_pcent_threshold def _include_change(self, change): ''' @@ -124,7 +137,9 @@ class BaseProvider(BaseSource): changes += extra if changes: - plan = Plan(existing, desired, changes) + plan = Plan(existing, desired, changes, + self.update_pcent_threshold, + self.delete_pcent_threshold) self.log.info('plan: %s', plan) return plan self.log.info('plan: No changes') diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index e44adc0..f761405 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -23,6 +23,8 @@ class HelperProvider(BaseProvider): self.__extra_changes = extra_changes self.apply_disabled = apply_disabled self.include_change_callback = include_change_callback + self.update_pcent_threshold = Plan.MAX_SAFE_UPDATE_PCENT + self.delete_pcent_threshold = Plan.MAX_SAFE_DELETE_PCENT def populate(self, zone, target=False, lenient=False): pass From 50ac2f794cf01da8ac0cc48e5e462b019592da9a Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 10 Oct 2017 14:39:25 -0700 Subject: [PATCH 77/84] add tests --- tests/test_octodns_provider_base.py | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index f761405..fde1396 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -290,3 +290,59 @@ class TestBaseProvider(TestCase): Plan.MAX_SAFE_DELETE_PCENT))] Plan(zone, zone, changes).raise_if_unsafe() + + def test_safe_updates_min_existing_override(self): + safe_pcent = .4 + # 40% + 1 fails when more + # than MIN_EXISTING_RECORDS exist + zone = Zone('unit.tests.', []) + record = Record.new(zone, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }) + + for i in range(int(Plan.MIN_EXISTING_RECORDS)): + zone.add_record(Record.new(zone, str(i), { + 'ttl': 60, + 'type': 'A', + 'value': '2.3.4.5' + })) + + changes = [Update(record, record) + for i in range(int(Plan.MIN_EXISTING_RECORDS * + safe_pcent) + 1)] + + with self.assertRaises(UnsafePlan) as ctx: + Plan(zone, zone, changes, + update_pcent_threshold=safe_pcent).raise_if_unsafe() + + self.assertTrue('Too many updates' in ctx.exception.message) + + def test_safe_deletes_min_existing_override(self): + safe_pcent = .4 + # 40% + 1 fails when more + # than MIN_EXISTING_RECORDS exist + zone = Zone('unit.tests.', []) + record = Record.new(zone, 'a', { + 'ttl': 30, + 'type': 'A', + 'value': '1.2.3.4', + }) + + for i in range(int(Plan.MIN_EXISTING_RECORDS)): + zone.add_record(Record.new(zone, str(i), { + 'ttl': 60, + 'type': 'A', + 'value': '2.3.4.5' + })) + + changes = [Delete(record) + for i in range(int(Plan.MIN_EXISTING_RECORDS * + safe_pcent) + 1)] + + with self.assertRaises(UnsafePlan) as ctx: + Plan(zone, zone, changes, + delete_pcent_threshold=safe_pcent).raise_if_unsafe() + + self.assertTrue('Too many deletes' in ctx.exception.message) From 3562b0dd4cf3abc5bc84d5a7abf38ac2c7766985 Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 10 Oct 2017 15:05:58 -0700 Subject: [PATCH 78/84] log this in init --- octodns/provider/base.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 86175e7..f6ff1b7 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -59,11 +59,6 @@ class Plan(object): update_pcent = self.change_counts['Update'] / existing_record_count delete_pcent = self.change_counts['Delete'] / existing_record_count - self.log.debug('raise_if_unsafe: update_pcent_threshold=%d, ' - 'delete_pcent_threshold=%d', - self.update_pcent_threshold, - self.delete_pcent_threshold) - if update_pcent > self.update_pcent_threshold: raise UnsafePlan('Too many updates, {} is over {} percent' '({}/{})'.format( @@ -92,8 +87,12 @@ class BaseProvider(BaseSource): update_pcent_threshold=Plan.MAX_SAFE_UPDATE_PCENT, delete_pcent_threshold=Plan.MAX_SAFE_DELETE_PCENT): super(BaseProvider, self).__init__(id) - self.log.debug('__init__: id=%s, apply_disabled=%s', id, - apply_disabled) + self.log.debug('__init__: id=%s, apply_disabled=%s, ' + 'update_pcent_threshold=%d, delete_pcent_threshold=%d', + id, + apply_disabled, + update_pcent_threshold, + delete_pcent_threshold) self.apply_disabled = apply_disabled self.update_pcent_threshold = update_pcent_threshold self.delete_pcent_threshold = delete_pcent_threshold From ffeceb39b19ab85789e83cb2b72ea2cdba3f1425 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Fri, 13 Oct 2017 13:15:24 -0700 Subject: [PATCH 79/84] Handle Manager.dump with an empty Zone --- octodns/manager.py | 4 +++- tests/test_octodns_manager.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/octodns/manager.py b/octodns/manager.py index 8439eb6..36a3592 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -11,7 +11,7 @@ from importlib import import_module from os import environ import logging -from .provider.base import BaseProvider +from .provider.base import BaseProvider, Plan from .provider.yaml import YamlProvider from .record import Record from .yaml import safe_load @@ -362,6 +362,8 @@ class Manager(object): source.populate(zone, lenient=lenient) plan = target.plan(zone) + if plan is None: + plan = Plan(zone, zone, []) target.apply(plan) def validate_configs(self): diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 45a3b55..a5f2022 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -11,6 +11,7 @@ from unittest import TestCase from octodns.record import Record from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager +from octodns.yaml import safe_load from octodns.zone import Zone from helpers import GeoProvider, NoSshFpProvider, SimpleProvider, \ @@ -211,6 +212,17 @@ class TestManager(TestCase): with self.assertRaises(IOError): manager.dump('unknown.zone.', tmpdir.dirname, False, 'in') + def test_dump_empty(self): + with TemporaryDirectory() as tmpdir: + environ['YAML_TMP_DIR'] = tmpdir.dirname + manager = Manager(get_config_filename('simple.yaml')) + + manager.dump('empty.', tmpdir.dirname, False, 'in') + + with open(join(tmpdir.dirname, 'empty.yaml')) as fh: + data = safe_load(fh, False) + self.assertFalse(data) + def test_validate_configs(self): Manager(get_config_filename('simple-validate.yaml')).validate_configs() From f45ff51062ef5377c4c9b01edb0098e5dc0a78eb Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Sat, 14 Oct 2017 08:06:06 +0200 Subject: [PATCH 80/84] Fix various logging lines in GoogleCloudProvider. --- octodns/provider/googlecloud.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index c544d63..370c750 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -106,7 +106,6 @@ class GoogleCloudProvider(BaseProvider): for i in range(120): gcloud_changes.reload() - self.log.debug("Waiting for changes to complete") # https://cloud.google.com/dns/api/v1/changes#resource # status can be one of either "pending" or "done" if gcloud_changes.status != 'pending': @@ -154,8 +153,7 @@ class GoogleCloudProvider(BaseProvider): # add this new zone to the list of zones. self._gcloud_zones[gcloud_zone.dns_name] = gcloud_zone - self.log.info("Created zone %s. Fqdn %s." % - (zone_name, dns_name)) + self.log.info("Created zone {}. Fqdn {}.".format(zone_name, dns_name)) return gcloud_zone From 7958618f63a3ecf541a228061b901ba6b02e34d9 Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Sat, 14 Oct 2017 19:32:24 +0200 Subject: [PATCH 81/84] Use uuid4 for zone name in GoogleCloudProvider use uuid4().hex to ensure unique zone_name generation and thereby streamline with the other providers. --- octodns/provider/googlecloud.py | 19 +++------------- tests/test_octodns_provider_googlecloud.py | 26 ---------------------- 2 files changed, 3 insertions(+), 42 deletions(-) diff --git a/octodns/provider/googlecloud.py b/octodns/provider/googlecloud.py index 370c750..6ca0794 100644 --- a/octodns/provider/googlecloud.py +++ b/octodns/provider/googlecloud.py @@ -5,10 +5,10 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import re import shlex import time from logging import getLogger +from uuid import uuid4 from google.cloud import dns @@ -128,21 +128,8 @@ class GoogleCloudProvider(BaseProvider): """ # Zone name must begin with a letter, end with a letter or digit, # and only contain lowercase letters, digits or dashes - zone_name = re.sub("[^a-z0-9-]", "", - dns_name[:-1].replace('.', "-")) - - # Check if there is another zone in google cloud which has the same - # name as the new one - while zone_name in [z.name for z in self.gcloud_zones.values()]: - # If there is a zone in google cloud alredy, then try suffixing the - # name with a -i where i is a number which keeps increasing until - # a free name has been reached. - m = re.match("^(.+)-([0-9]+$)", zone_name) - if m: - i = int(m.group(2)) + 1 - zone_name = "{}-{!s}".format(m.group(1), i) - else: - zone_name += "-2" + zone_name = '{}-{}'.format( + dns_name[:-1].replace('.', '-'), uuid4().hex) gcloud_zone = self.gcloud_client.zone( name=zone_name, diff --git a/tests/test_octodns_provider_googlecloud.py b/tests/test_octodns_provider_googlecloud.py index c2e976c..adc2112 100644 --- a/tests/test_octodns_provider_googlecloud.py +++ b/tests/test_octodns_provider_googlecloud.py @@ -427,29 +427,3 @@ class TestGoogleCloudProvider(TestCase): mock_zone.create.assert_called() provider.gcloud_client.zone.assert_called() - provider.gcloud_client.zone.assert_called_once_with( - dns_name=u'nonexistant.zone.mock', name=u'nonexistant-zone-moc') - - def test__create_zone_with_duplicate_names(self): - - def _create_dummy_zone(name, dns_name): - return DummyGoogleCloudZone(name=name, dns_name=dns_name) - - provider = self._get_provider() - - provider.gcloud_client = Mock() - provider.gcloud_client.zone = Mock(side_effect=_create_dummy_zone) - provider.gcloud_client.list_zones = Mock( - return_value=DummyIterator([])) - - _gcloud_zones = { - 'unit-tests': DummyGoogleCloudZone("a.unit-tests.", "unit-tests") - } - - provider._gcloud_zones = _gcloud_zones - - test_zone_1 = provider._create_gcloud_zone("unit.tests.") - self.assertEqual(test_zone_1.name, "unit-tests-2") - - test_zone_2 = provider._create_gcloud_zone("unit.tests.") - self.assertEqual(test_zone_2.name, "unit-tests-3") From d2e1aafa8588ba8023e3c04a5785ecd3c3effd1d Mon Sep 17 00:00:00 2001 From: Nicolas KAROLAK Date: Mon, 16 Oct 2017 18:49:28 +0200 Subject: [PATCH 82/84] use . instead of source [SC2039] --- .git_hooks_pre-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.git_hooks_pre-commit b/.git_hooks_pre-commit index 9cb854a..c17906b 100755 --- a/.git_hooks_pre-commit +++ b/.git_hooks_pre-commit @@ -6,6 +6,6 @@ HOOKS=`dirname $0` GIT=`dirname $HOOKS` ROOT=`dirname $GIT` -source $ROOT/env/bin/activate +. $ROOT/env/bin/activate $ROOT/script/lint $ROOT/script/test From 8352ab89efb02b7bab9138b1e11a4de8c99e6e70 Mon Sep 17 00:00:00 2001 From: Tim Hughes Date: Tue, 17 Oct 2017 16:16:17 +0100 Subject: [PATCH 83/84] adds warning to dyn provider when it cannot load a trafficdirector --- octodns/provider/dyn.py | 3 ++- tests/test_octodns_provider_dyn.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 721b3a7..65acf1d 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -290,7 +290,8 @@ class DynProvider(BaseProvider): for td in get_all_dsf_services(): try: fqdn, _type = td.label.split(':', 1) - except ValueError: + except ValueError as e: + self.log.warn("Failed to load TraficDirector '{}': {}".format(td.label, e)) continue tds[fqdn][_type] = td self._traffic_directors = dict(tds) diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 9be253d..5e4c86b 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -8,7 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from dyn.tm.errors import DynectGetError from dyn.tm.services.dsf import DSFResponsePool from json import loads -from mock import MagicMock, call, patch +from mock import MagicMock, call, patch, Mock from unittest import TestCase from octodns.record import Create, Delete, Record, Update @@ -601,6 +601,7 @@ class TestDynProviderGeo(TestCase): provider = DynProvider('test', 'cust', 'user', 'pass', True) # short-circuit session checking provider._dyn_sess = True + provider.log.warn = MagicMock() # no tds mock.side_effect = [{'data': []}] @@ -649,6 +650,8 @@ class TestDynProviderGeo(TestCase): set(tds.keys())) self.assertEquals(['A'], tds['unit.tests.'].keys()) self.assertEquals(['A'], tds['geo.unit.tests.'].keys()) + provider.log.warn.assert_called_with("Failed to load TraficDirector 'something else': need more than 1 value to unpack") + @patch('dyn.core.SessionEngine.execute') def test_traffic_director_monitor(self, mock): From f39e1d28c810138daa62a86cafdc39492bcad670 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 18 Oct 2017 10:21:10 -0700 Subject: [PATCH 84/84] Fix log formatting and lint compliance --- octodns/provider/dyn.py | 3 ++- tests/test_octodns_provider_dyn.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/octodns/provider/dyn.py b/octodns/provider/dyn.py index 65acf1d..010fd31 100644 --- a/octodns/provider/dyn.py +++ b/octodns/provider/dyn.py @@ -291,7 +291,8 @@ class DynProvider(BaseProvider): try: fqdn, _type = td.label.split(':', 1) except ValueError as e: - self.log.warn("Failed to load TraficDirector '{}': {}".format(td.label, e)) + self.log.warn("Failed to load TraficDirector '%s': %s", + td.label, e.message) continue tds[fqdn][_type] = td self._traffic_directors = dict(tds) diff --git a/tests/test_octodns_provider_dyn.py b/tests/test_octodns_provider_dyn.py index 5e4c86b..4415347 100644 --- a/tests/test_octodns_provider_dyn.py +++ b/tests/test_octodns_provider_dyn.py @@ -8,7 +8,7 @@ from __future__ import absolute_import, division, print_function, \ from dyn.tm.errors import DynectGetError from dyn.tm.services.dsf import DSFResponsePool from json import loads -from mock import MagicMock, call, patch, Mock +from mock import MagicMock, call, patch from unittest import TestCase from octodns.record import Create, Delete, Record, Update @@ -650,8 +650,10 @@ class TestDynProviderGeo(TestCase): set(tds.keys())) self.assertEquals(['A'], tds['unit.tests.'].keys()) self.assertEquals(['A'], tds['geo.unit.tests.'].keys()) - provider.log.warn.assert_called_with("Failed to load TraficDirector 'something else': need more than 1 value to unpack") - + provider.log.warn.assert_called_with("Failed to load TraficDirector " + "'%s': %s", 'something else', + 'need more than 1 value to ' + 'unpack') @patch('dyn.core.SessionEngine.execute') def test_traffic_director_monitor(self, mock):