From 0e20c076b002ef87f3a7d12b2c791d55b6e4c05c Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 16 Jun 2017 14:20:36 -0700 Subject: [PATCH 01/24] First skeleton of Azure DNS Provider class --- octodns/provider/azuredns.py | 62 ++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 octodns/provider/azuredns.py diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py new file mode 100644 index 0000000..17e93b9 --- /dev/null +++ b/octodns/provider/azuredns.py @@ -0,0 +1,62 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from azure.common.credentials import ServicePrincipalCredentials +from azure.mgmt.dns import DnsManagementClient + + +from collections import defaultdict +# from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. +import logging +import re + +from ..record import Record, Update +from .base import BaseProvider + +class AzureProvider(BaseProvider): + ''' + Azure DNS Provider + + azure.py: + class: octodns.provider.azure.AzureProvider + # Current support of authentication of access to Azure services only + # includes using a Service Principal: + # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ + # resource-group-create-service-principal-portal + # The Azure Active Directory Application ID (referred to client ID) req: + client_id: + # Authentication Key Value req: + key: + # Directory ID (referred to tenant ID) req: + directory_id: + # Subscription ID req: + sub_id: + # Resource Group name req: + resource_group: + + testing: test authentication vars located in /home/t-hehwan/vars.txt + ''' + + # TODO. Will add support as project progresses. + SUPPORTS_GEO = False + + def __init__(self, id, client_id, key, directory_id, sub_id, \ + resource_group, *args, **kwargs): + self.log = logging.getLogger('AzureProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, client_id=%s, ' + 'key=***, directory_id:%s', id, client_id, directory_id) + super(AzureProvider, self).__init__(id, *args, **kwargs) + + credentials = ServicePrincipalCredentials( + client_id = client_id, secret = key, tenant = directory_id + ) + self._dns_client = DnsManagementClient(credentials, sub_id) + self._resource_group = resource_group + + + def _apply(self, plan): + From 386ada34f0750922a930fc0660f668d3552d6aae Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 19 Jun 2017 17:04:25 -0700 Subject: [PATCH 02/24] Added onto azuredns.py. Still completing code skeleton --- octodns/provider/azuredns.py | 116 +++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 13 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 17e93b9..8e1b992 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient - +from azure.mgmt.dns.models import * from collections import defaultdict # from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. @@ -17,6 +17,23 @@ import re from ..record import Record, Update from .base import BaseProvider +# Only made for A records. will have to adjust for more generic params types +class _AzureRecord(object): + def __init__(self, resource_group_name, record, values=None) + self.resource_group_name = resource_group_name + self.zone_name = record.zone.name + self.relative_record_set_name = record.name + self.record_type = record._type + + type_name = '{}records'.format(self.record_type) + class_name = '{}'.format(self.record_type).capitalize() + + 'Record'.format(self.record_type) + _values = [record._process_values] + self.params = {'ttl':record.ttl or 1800, \ + type_name:[eval(class_name)(value) for value in _values] or []} + + + class AzureProvider(BaseProvider): ''' Azure DNS Provider @@ -44,19 +61,92 @@ class AzureProvider(BaseProvider): # TODO. Will add support as project progresses. SUPPORTS_GEO = False - def __init__(self, id, client_id, key, directory_id, sub_id, \ - resource_group, *args, **kwargs): - self.log = logging.getLogger('AzureProvider[{}]'.format(id)) - self.log.debug('__init__: id=%s, client_id=%s, ' + def __init__(self, id, client_id, key, directory_id, sub_id, \ + resource_group, *args, **kwargs): + self.log = logging.getLogger('AzureProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, client_id=%s, ' 'key=***, directory_id:%s', id, client_id, directory_id) - super(AzureProvider, self).__init__(id, *args, **kwargs) + super(AzureProvider, self).__init__(id, *args, **kwargs) - credentials = ServicePrincipalCredentials( - client_id = client_id, secret = key, tenant = directory_id - ) - self._dns_client = DnsManagementClient(credentials, sub_id) - self._resource_group = resource_group + credentials = ServicePrincipalCredentials( + client_id = client_id, secret = key, tenant = directory_id + ) + self._dns_client = DnsManagementClient(credentials, sub_id) + self._resource_group = resource_group + + + self._azure_zones = None + self._azure_records = {} # this is populated through populate() + + # TODO: health checks a la route53. - def _apply(self, plan): - + + # TODO: add support for all types. First skeleton: add A. + def supports(self, record): + return record._type == 'A' + + @property + def azure_zones(self): + # TODO: return zones. will be created by populate() + + # Given a zone name, returns the zone id. If DNE, creates it. + def _get_zone_id(self, name): + + + def populate(self, zone, target): + self._azure_records = {} + + for record in zone.records: + + + def _apply_Create(self, change): + new = change.new + ar = self._get_azure_record(new) + + create = self._dns_client.record_sets.create_or_update + create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name \ + ar.record_type, ar.params) + + # type plan: Plan class from .base + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + + # validate that the zone exists. function creates zone if DNE. + self._get_zone_id(desired.name) + + + + # Some parsing bits to call _mod_Create or _mod_Delete. + # changes is a list of Delete and Create objects. + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + + + # ********** + # Figuring out what object plan is. + + # self._executor = ThreadPoolExecutor(max_workers) + # futures.append(self._executor.submit(self._populate_and_plan, + # zone_name, sources, targets)) + # plans = [p for f in futures for p in f.results()] + # type of plans[0] == type of one output of _populate_and_plan + + # for target, plan in plans: + # apply(plan) + + + # type(target) == BaseProvider + # type(plan) == Plan() + + # Plan(existing, desired, changes) + # existing.type == desired.type == Zone(desired.name, desired.sub_zones) + # Zone(name, sub_zones) (str and set of strs) + # changes.type = [Delete/Create] + \ No newline at end of file From ae9dd97f16e06152d27f28098b56a0f91eb476a5 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 20 Jun 2017 14:43:51 -0700 Subject: [PATCH 03/24] Filled out skeleton. Starting Testing --- .gitignore | 3 ++ MakeFile | 5 +++ octodns/provider/azuredns.py | 66 ++++++++++++++++++++++++++++++------ 3 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 MakeFile diff --git a/.gitignore b/.gitignore index 842a688..eca95c9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ nosetests.xml octodns.egg-info/ output/ tmp/ +Makefile +build/ +config/ \ No newline at end of file diff --git a/MakeFile b/MakeFile new file mode 100644 index 0000000..53fc947 --- /dev/null +++ b/MakeFile @@ -0,0 +1,5 @@ +local-rebuild: + sudo rm -r build + sudo rm -r octodns.egg-info/ + sudo python setup.py build -q + sudo python setup.py install -q \ No newline at end of file diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 8e1b992..2c55d7f 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -19,14 +19,14 @@ from .base import BaseProvider # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group_name, record, values=None) + def __init__(self, resource_group_name, record, values=None): self.resource_group_name = resource_group_name self.zone_name = record.zone.name self.relative_record_set_name = record.name self.record_type = record._type type_name = '{}records'.format(self.record_type) - class_name = '{}'.format(self.record_type).capitalize() + + class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) _values = [record._process_values] self.params = {'ttl':record.ttl or 1800, \ @@ -75,30 +75,76 @@ class AzureProvider(BaseProvider): self._resource_group = resource_group - self._azure_zones = None - self._azure_records = {} # this is populated through populate() + self._azure_zones = None # will be a dictionary. key: name. val: id. + self._azure_records = None # will be dict by octodns record, az record + self._supported_types = ['A'] + # TODO: health checks a la route53. # TODO: add support for all types. First skeleton: add A. def supports(self, record): - return record._type == 'A' + # TODO: possibly refactor + return record._type in self._supported_types @property def azure_zones(self): - # TODO: return zones. will be created by populate() + if self._azure_zones is None: + self.log.debug('azure_zones: loading') + zones = {} + for zone in self._dns_client.zones.list(): + zones[zone['name']] = zone['id'] + self._azure_zones = zones + return self._azure_zones # Given a zone name, returns the zone id. If DNE, creates it. - def _get_zone_id(self, name): - + def _get_zone_id(self, name, create=False): + self.log.debug('_get_zone_id: name=%s', name) + if name in self.azure_zones: + id = self.azure_zones[name] + self.log.debug('_get_zone_id: id=%s', id) + return id + if create: + #TODO + return None + # Create a dictionary of record objects by zone and octodns record names + # TODO: add geo parsing def populate(self, zone, target): - self._azure_records = {} + self.log.debug('populate: name=%s', zone.name) + before = len(zone.records) - for record in zone.records: + zone_id = self._get_zone_id(zone.name) + if zone_id: + records = defaultdict(list) + for type in self._supported_types: + for azrecord in self.dns_client.record_sets.list_by_type(self._resource_group, zone.name, type): + record_name = azrecord.name + data = getattr(self, '_data_for_{}'.format(type))(type, azrecord) + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) + self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) + + self.log.info('populate: found %s records', len(zone.records)-before) + + # might not need + def _get_type(azrecord): + azrecord['type'].split('/')[-1] + + def _data_for_A(self, type, azrecord): + return { + 'type': type + 'ttl': azrecord['ttl'], + 'values': [ar.ipv4_address for ar in azrecord.arecords] + } + def _get_azure_record(record): + try: + return self._azure_records[record] + except: + raise def _apply_Create(self, change): new = change.new From f48ef28688cc7be2211e42d4a39614fbb4cccb40 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 20 Jun 2017 14:55:59 -0700 Subject: [PATCH 04/24] Added shell script --- MakeFile | 5 ----- octodns/provider/azuredns.py | 3 +++ rb.txt | 7 +++++++ 3 files changed, 10 insertions(+), 5 deletions(-) delete mode 100644 MakeFile create mode 100755 rb.txt diff --git a/MakeFile b/MakeFile deleted file mode 100644 index 53fc947..0000000 --- a/MakeFile +++ /dev/null @@ -1,5 +0,0 @@ -local-rebuild: - sudo rm -r build - sudo rm -r octodns.egg-info/ - sudo python setup.py build -q - sudo python setup.py install -q \ No newline at end of file diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 2c55d7f..da348c1 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -17,6 +17,9 @@ import re from ..record import Record, Update from .base import BaseProvider + +#TODO: changes made to master include adding /build, Makefile to .gitignore and +# making Makefile. # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): def __init__(self, resource_group_name, record, values=None): diff --git a/rb.txt b/rb.txt new file mode 100755 index 0000000..30d33a1 --- /dev/null +++ b/rb.txt @@ -0,0 +1,7 @@ +#!/bin/bash +#script to rebuild octodns quickly + +sudo rm -r /home/t-hehwan/GitHub/octodns/build +sudo rm -r /home/t-hehwan/GitHub/octodns/octodns.egg-info +sudo python /home/t-hehwan/GitHub/octodns/setup.py build -q +sudo python /home/t-hehwan/GitHub/octodns/setup.py install -q From f5bce43e1054fe310dc77e379797149360a93326 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 20 Jun 2017 15:54:35 -0700 Subject: [PATCH 05/24] Testing AzureProvider. TODO: resolve 'Exception: Unknown provider class: octodns.provider.azure.AzureProvider' --- octodns/provider/azuredns.py | 85 ++++++++++++++++++------------------ rb.txt | 5 ++- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index da348c1..f1b342a 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -4,7 +4,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals - + from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient from azure.mgmt.dns.models import * @@ -13,10 +13,13 @@ from collections import defaultdict # from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. import logging import re +from ..record import Record, Up -from ..record import Record, Update from .base import BaseProvider +class A(BaseProvider): + def __init__(self): + pass #TODO: changes made to master include adding /build, Makefile to .gitignore and # making Makefile. @@ -38,60 +41,56 @@ class _AzureRecord(object): class AzureProvider(BaseProvider): - ''' - Azure DNS Provider - - azure.py: - class: octodns.provider.azure.AzureProvider - # Current support of authentication of access to Azure services only - # includes using a Service Principal: - # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ - # resource-group-create-service-principal-portal - # The Azure Active Directory Application ID (referred to client ID) req: - client_id: - # Authentication Key Value req: - key: - # Directory ID (referred to tenant ID) req: - directory_id: - # Subscription ID req: - sub_id: - # Resource Group name req: - resource_group: - - testing: test authentication vars located in /home/t-hehwan/vars.txt - ''' - - # TODO. Will add support as project progresses. - SUPPORTS_GEO = False - - def __init__(self, id, client_id, key, directory_id, sub_id, \ - resource_group, *args, **kwargs): + ''' + Azure DNS Provider + + azure.py: + class: octodns.provider.azure.AzureProvider + # Current support of authentication of access to Azure services only + # includes using a Service Principal: + # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ + # resource-group-create-service-principal-portal + # The Azure Active Directory Application ID (referred to client ID) req: + client_id: + # Authentication Key Value req: + key: + # Directory ID (referred to tenant ID) req: + directory_id: + # Subscription ID req: + sub_id: + # Resource Group name req: + resource_group: + + testing: test authentication vars located in /home/t-hehwan/vars.txt + ''' + SUPPORTS_GEO = False # TODO. Will add support as project progresses. + + def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, client_id=%s, ' - 'key=***, directory_id:%s', id, client_id, directory_id) + 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) - + credentials = ServicePrincipalCredentials( - client_id = client_id, secret = key, tenant = directory_id + client_id = client_id, secret = key, tenant = directory_id ) self._dns_client = DnsManagementClient(credentials, sub_id) self._resource_group = resource_group - self._azure_zones = None # will be a dictionary. key: name. val: id. self._azure_records = None # will be dict by octodns record, az record - + self._supported_types = ['A'] + + # TODO: health checks a la route53. + + - # TODO: health checks a la route53. - - - # TODO: add support for all types. First skeleton: add A. def supports(self, record): # TODO: possibly refactor return record._type in self._supported_types - + @property def azure_zones(self): if self._azure_zones is None: @@ -110,6 +109,7 @@ class AzureProvider(BaseProvider): self.log.debug('_get_zone_id: id=%s', id) return id if create: + raise Exception #TODO return None @@ -138,7 +138,7 @@ class AzureProvider(BaseProvider): def _data_for_A(self, type, azrecord): return { - 'type': type + 'type': type, 'ttl': azrecord['ttl'], 'values': [ar.ipv4_address for ar in azrecord.arecords] } @@ -154,8 +154,7 @@ class AzureProvider(BaseProvider): ar = self._get_azure_record(new) create = self._dns_client.record_sets.create_or_update - create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name \ - ar.record_type, ar.params) + create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name, ar.record_type, ar.params) # type plan: Plan class from .base def _apply(self, plan): diff --git a/rb.txt b/rb.txt index 30d33a1..d6a7949 100755 --- a/rb.txt +++ b/rb.txt @@ -3,5 +3,6 @@ sudo rm -r /home/t-hehwan/GitHub/octodns/build sudo rm -r /home/t-hehwan/GitHub/octodns/octodns.egg-info -sudo python /home/t-hehwan/GitHub/octodns/setup.py build -q -sudo python /home/t-hehwan/GitHub/octodns/setup.py install -q +sudo python /home/t-hehwan/GitHub/octodns/setup.py -q build +sudo python /home/t-hehwan/GitHub/octodns/setup.py -q install +octodns-sync --config-file=./config/production.yaml \ No newline at end of file From 92828ce1c6a99c6d29fe2b223cc9e938a404fae1 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Wed, 21 Jun 2017 14:21:14 -0700 Subject: [PATCH 06/24] Successfully able to add A records. TODO: check against live server to remove records not listed in config --- octodns/provider/azuredns.py | 85 +++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index f1b342a..3a442e8 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals +import sys from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient @@ -13,32 +14,41 @@ from collections import defaultdict # from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. import logging import re -from ..record import Record, Up - +from ..record import Record, Update from .base import BaseProvider -class A(BaseProvider): - def __init__(self): - pass #TODO: changes made to master include adding /build, Makefile to .gitignore and # making Makefile. # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group_name, record, values=None): - self.resource_group_name = resource_group_name - self.zone_name = record.zone.name - self.relative_record_set_name = record.name + def __init__(self, resource_group, record, values=None): + self.resource_group = resource_group + self.zone_name = record.zone.name[0:len(record.zone.name)-1] # strips last period + self.relative_record_set_name = record.name or '@' self.record_type = record._type - type_name = '{}records'.format(self.record_type) + type_name = '{}records'.format(self.record_type).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) - _values = [record._process_values] - self.params = {'ttl':record.ttl or 1800, \ - type_name:[eval(class_name)(value) for value in _values] or []} + data = values or record.data #This should fail if it gets to record.data? It only returns ttl. TODO + print('findme: ',file=sys.stderr) + print(data, file=sys.stderr) + print('\n',file=sys.stderr) + #depending on mult values or not + self.params = {} + try: + self.params = {'ttl':record.ttl or 1800, \ + type_name:[eval(class_name)(ip) for ip in data['values']] or []} + except KeyError: # means that doesn't have multiple values but single value + self.params = {'ttl':record.ttl or 1800, \ + type_name:[eval(class_name)(data['value'])] or []} + + # ar = _AzureRecord(self._resource_group, new, new.data) + + class AzureProvider(BaseProvider): ''' @@ -96,8 +106,8 @@ class AzureProvider(BaseProvider): if self._azure_zones is None: self.log.debug('azure_zones: loading') zones = {} - for zone in self._dns_client.zones.list(): - zones[zone['name']] = zone['id'] + for zone in self._dns_client.zones.list_by_resource_group(self._resource_group): + zones[zone.name] = zone.id self._azure_zones = zones return self._azure_zones @@ -109,8 +119,9 @@ class AzureProvider(BaseProvider): self.log.debug('_get_zone_id: id=%s', id) return id if create: + self.log.debug('_get_zone_id: no matching zone; creating %s', name) raise Exception - #TODO + #TODO, write code return None # Create a dictionary of record objects by zone and octodns record names @@ -128,7 +139,7 @@ class AzureProvider(BaseProvider): data = getattr(self, '_data_for_{}'.format(type))(type, azrecord) record = Record.new(zone, record_name, data, source=self) zone.add_record(record) - self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) + self._azure_records[record] = _AzureRecord(self._resource_group, record, data) self.log.info('populate: found %s records', len(zone.records)-before) @@ -137,24 +148,48 @@ class AzureProvider(BaseProvider): azrecord['type'].split('/')[-1] def _data_for_A(self, type, azrecord): + print('xxxxx\n',file=sys.stderr) + for ar in azrecords.arecords: + print(ar,file=sys.stderr) + + + v = [ARecord(ar.ipv4_address) for ar in azrecord.arecords] + # print(v, file=sys.stderr) + # print('99999999999999999999999999999999999999999999999999\n',file=sys.stderr) return { 'type': type, 'ttl': azrecord['ttl'], - 'values': [ar.ipv4_address for ar in azrecord.arecords] + 'value': v } - def _get_azure_record(record): - try: - return self._azure_records[record] - except: - raise + # def _get_azure_record(self, record): + # try: + # return self._azure_records[record] + # except: + # self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) + # return self._azure_records[record] + # except: + # raise def _apply_Create(self, change): new = change.new - ar = self._get_azure_record(new) + + #validate that the zone exists. + #self._get_zone_id(new.name, create=True) + + #ar = self._get_azure_record(new) + ar = _AzureRecord(self._resource_group, new, new.data) create = self._dns_client.record_sets.create_or_update - create(ar.resource_group_name, ar.zone_name, ar.relative_record_set_name, ar.record_type, ar.params) + print('find:{} {} {} {} {}\n'.format(ar.resource_group,ar.zone_name,ar.relative_record_set_name,ar.record_type,ar.params), file=sys.stderr) + for arec in ar.params['arecords']: + print(str(arec.ipv4_address) + ', ', file=sys.stderr) + print('\n',file=sys.stderr) + create(resource_group_name=ar.resource_group, + zone_name=ar.zone_name, + relative_record_set_name=ar.relative_record_set_name, + record_type=ar.record_type, + parameters=ar.params) # type plan: Plan class from .base def _apply(self, plan): From 3c1e409e6fea2347c4cd5d18c6fccc84387ae797 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Thu, 22 Jun 2017 14:22:40 -0700 Subject: [PATCH 07/24] Added support for CNAME, AAAA, MX, SRV, NS, PTR. TODO: add TXT. add zone creation. create tests --- octodns/provider/azuredns.py | 162 ++++++++++++++++++++++------------- 1 file changed, 101 insertions(+), 61 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 3a442e8..66ca8e3 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -22,7 +22,8 @@ from .base import BaseProvider # making Makefile. # Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group, record, values=None): + def __init__(self, resource_group, record, values=None, ttl=1800): + # print('Here4',file=sys.stderr) self.resource_group = resource_group self.zone_name = record.zone.name[0:len(record.zone.name)-1] # strips last period self.relative_record_set_name = record.name or '@' @@ -31,22 +32,21 @@ class _AzureRecord(object): type_name = '{}records'.format(self.record_type).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) + if values == None: + return + # TODO: clean up this bit. data = values or record.data #This should fail if it gets to record.data? It only returns ttl. TODO - print('findme: ',file=sys.stderr) - print(data, file=sys.stderr) - print('\n',file=sys.stderr) #depending on mult values or not + #TODO: import explicitly. eval() uses for example ARecord from azure.mgmt.dns.models self.params = {} try: - self.params = {'ttl':record.ttl or 1800, \ + self.params = {'ttl':record.ttl or ttl, \ type_name:[eval(class_name)(ip) for ip in data['values']] or []} except KeyError: # means that doesn't have multiple values but single value - self.params = {'ttl':record.ttl or 1800, \ + self.params = {'ttl':record.ttl or ttl, \ type_name:[eval(class_name)(data['value'])] or []} - - # ar = _AzureRecord(self._resource_group, new, new.data) @@ -88,17 +88,17 @@ class AzureProvider(BaseProvider): self._resource_group = resource_group self._azure_zones = None # will be a dictionary. key: name. val: id. - self._azure_records = None # will be dict by octodns record, az record + self._azure_records = {} # will be dict by octodns record, az record - self._supported_types = ['A'] + self._supported_types = ['CNAME', 'A', 'AAAA', 'MX', 'SRV', 'NS', 'PTR'] + # TODO: add TXT - # TODO: health checks a la route53. + # TODO: health checks a la route53. # TODO: add support for all types. First skeleton: add A. def supports(self, record): - # TODO: possibly refactor return record._type in self._supported_types @property @@ -114,82 +114,121 @@ class AzureProvider(BaseProvider): # Given a zone name, returns the zone id. If DNE, creates it. def _get_zone_id(self, name, create=False): self.log.debug('_get_zone_id: name=%s', name) - if name in self.azure_zones: - id = self.azure_zones[name] + try: + id = self._dns_client.zones.get(self._resource_group, name) self.log.debug('_get_zone_id: id=%s', id) return id - if create: - self.log.debug('_get_zone_id: no matching zone; creating %s', name) - raise Exception - #TODO, write code - return None - + except: + if create: + self.log.debug('_get_zone_id: no matching zone; creating %s', name) + #TODO: write + return None #placeholder + return None + # Create a dictionary of record objects by zone and octodns record names # TODO: add geo parsing def populate(self, zone, target): - self.log.debug('populate: name=%s', zone.name) + zone_name = zone.name[0:len(zone.name)-1]#Azure zone names do not include suffix . + self.log.debug('populate: name=%s', zone_name) before = len(zone.records) - - zone_id = self._get_zone_id(zone.name) + zone_id = self._get_zone_id(zone_name) if zone_id: - records = defaultdict(list) + #records = defaultdict(list) for type in self._supported_types: - for azrecord in self.dns_client.record_sets.list_by_type(self._resource_group, zone.name, type): - record_name = azrecord.name - data = getattr(self, '_data_for_{}'.format(type))(type, azrecord) + # print('populate. type: {}'.format(type),file=sys.stderr) + for azrecord in self._dns_client.record_sets.list_by_type(self._resource_group, zone_name, type): + # print(azrecord, file=sys.stderr) + record_name = azrecord.name if azrecord.name != '@' else '' + data = self._type_and_ttl(type, azrecord, + getattr(self, '_data_for_{}'.format(type))(azrecord)) # TODO: azure online interface allows None values. must validate. record = Record.new(zone, record_name, data, source=self) + # print('HERE0',file=sys.stderr) zone.add_record(record) self._azure_records[record] = _AzureRecord(self._resource_group, record, data) - + # print('HERE1',file=sys.stderr) self.log.info('populate: found %s records', len(zone.records)-before) - # might not need + # might not need def _get_type(azrecord): azrecord['type'].split('/')[-1] - def _data_for_A(self, type, azrecord): - print('xxxxx\n',file=sys.stderr) - for ar in azrecords.arecords: - print(ar,file=sys.stderr) - + def _type_and_ttl(self, type, azrecord, data): + data['type'] = type + data['ttl'] = azrecord.ttl + return data - v = [ARecord(ar.ipv4_address) for ar in azrecord.arecords] - # print(v, file=sys.stderr) - # print('99999999999999999999999999999999999999999999999999\n',file=sys.stderr) - return { - 'type': type, - 'ttl': azrecord['ttl'], - 'value': v + def _data_for_A(self, azrecord): + return {'values': [ar.ipv4_address for ar in azrecord.arecords]} + + def _data_for_AAAA(self, azrecord): + return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} + + def _data_for_TXT(self, azrecord): + print('azure',file=sys.stderr) + print([ar.value for ar in azrecord.txt_records], file=sys.stderr) + print('',file=sys.stderr) + return {'values': [ar.value for ar in azrecord.txt_records]} + + def _data_for_CNAME(self, azrecord): + try: + val = azrecord.cname_record.cname + if not val.endswith('.'): + val += '.' + return {'value': val} + except: + return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. + + def _data_for_PTR(self, azrecord): + try: + val = azrecord.ptr_records[0].ptdrname + if not val.endswith('.'): + val += '.' + return {'value': val} + except: + return {'value': '.' } #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. + + def _data_for_MX(self, azrecord): + return {'values': [{'priority':ar.preference, + 'value':ar.exchange} for ar in azrecord.mx_records]} + + def _data_for_SRV(self, azrecord): + return {'values': [{'priority': ar.priority, + 'weight': ar.weight, + 'port': ar.port, + 'target': ar.target} for ar in azrecord.srv_records] } - - # def _get_azure_record(self, record): - # try: - # return self._azure_records[record] - # except: - # self._azure_records[record] = _AzureRecord(self._resource_group, record, record.data) - # return self._azure_records[record] - # except: - # raise - + + def _data_for_NS(self, azrecord): + def period_validate(string): + return string if string.endswith('.') else string + '.' + vals = [ar.nsdname for ar in azrecord.ns_records] + return {'values': [period_validate(val) for val in vals]} + def _apply_Create(self, change): new = change.new - + #validate that the zone exists. #self._get_zone_id(new.name, create=True) - #ar = self._get_azure_record(new) ar = _AzureRecord(self._resource_group, new, new.data) create = self._dns_client.record_sets.create_or_update - print('find:{} {} {} {} {}\n'.format(ar.resource_group,ar.zone_name,ar.relative_record_set_name,ar.record_type,ar.params), file=sys.stderr) - for arec in ar.params['arecords']: - print(str(arec.ipv4_address) + ', ', file=sys.stderr) - print('\n',file=sys.stderr) create(resource_group_name=ar.resource_group, zone_name=ar.zone_name, relative_record_set_name=ar.relative_record_set_name, record_type=ar.record_type, parameters=ar.params) + + def _apply_Delete(self, change): + existing = change.existing + ar = _AzureRecord(self._resource_group, existing) + delete = self._dns_client.record_sets.delete + delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, + ar.record_type) + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) # type plan: Plan class from .base def _apply(self, plan): @@ -198,12 +237,9 @@ class AzureProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - # validate that the zone exists. function creates zone if DNE. self._get_zone_id(desired.name) - - # Some parsing bits to call _mod_Create or _mod_Delete. # changes is a list of Delete and Create objects. for change in changes: @@ -232,4 +268,8 @@ class AzureProvider(BaseProvider): # existing.type == desired.type == Zone(desired.name, desired.sub_zones) # Zone(name, sub_zones) (str and set of strs) # changes.type = [Delete/Create] - \ No newline at end of file + + + # Starts with sync in main() of sync. + # {u'values': ['3.3.3.3', '4.4.4.4'], u'type': 'A', u'ttl': 3600} + # {u'type': u'A', u'value': [u'3.3.3.3', u'4.4.4.4'], u'ttl': 3600L} \ No newline at end of file From 598acc943d48eca9ce0c461137d7dfeb242e9d7f Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 23 Jun 2017 15:19:27 -0700 Subject: [PATCH 08/24] Added full support of Azure DNS. TODO: testing. --- .gitignore | 5 +- doit.txt | 4 + octodns/provider/azuredns.py | 297 ++++++++++++++++++----------------- 3 files changed, 164 insertions(+), 142 deletions(-) create mode 100755 doit.txt diff --git a/.gitignore b/.gitignore index eca95c9..5a3f3f9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ nosetests.xml octodns.egg-info/ output/ tmp/ -Makefile build/ -config/ \ No newline at end of file +config/ +./rb.txt +./doit.txt diff --git a/doit.txt b/doit.txt new file mode 100755 index 0000000..1ff2da6 --- /dev/null +++ b/doit.txt @@ -0,0 +1,4 @@ +#!/bin/bash +#script to rebuild octodns quickly + +octodns-sync --config-file=./config/production.yaml --doit diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 66ca8e3..07b39fe 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -8,54 +8,66 @@ import sys from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient -from azure.mgmt.dns.models import * +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone + +from functools import reduce -from collections import defaultdict -# from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. import logging import re from ..record import Record, Update from .base import BaseProvider - -#TODO: changes made to master include adding /build, Makefile to .gitignore and -# making Makefile. -# Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group, record, values=None, ttl=1800): - # print('Here4',file=sys.stderr) + ''' + Wrapper for OctoDNS record. + azuredns.py: + class: octodns.provider.azuredns._AzureRecord + An _AzureRecord is easily accessible to the Azure DNS Management library + functions and is used to wrap all relevant data to create a record in + Azure. + ''' + + def __init__(self, resource_group, record, values=None): + ''' + :param resource_group: The name of resource group in Azure + :type resource_group: str + :param record: An OctoDNS record + :type record: ..record.Record + :param values: Parameters for a record. eg IP address, port, domain + name, etc. Values usually read from record.data + :type values: {'values': [...]} or {'value': [...]} + + :type return: _AzureRecord + ''' self.resource_group = resource_group - self.zone_name = record.zone.name[0:len(record.zone.name)-1] # strips last period + self.zone_name = record.zone.name[0:len(record.zone.name)-1] self.relative_record_set_name = record.name or '@' self.record_type = record._type - type_name = '{}records'.format(self.record_type).lower() + data = values or record.data + format_u_s = '' if record._type == 'A' else '_' + key_name ='{}{}records'.format(self.record_type, format_u_s).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) - if values == None: - return - # TODO: clean up this bit. - data = values or record.data #This should fail if it gets to record.data? It only returns ttl. TODO - - #depending on mult values or not - #TODO: import explicitly. eval() uses for example ARecord from azure.mgmt.dns.models - self.params = {} - try: - self.params = {'ttl':record.ttl or ttl, \ - type_name:[eval(class_name)(ip) for ip in data['values']] or []} - except KeyError: # means that doesn't have multiple values but single value - self.params = {'ttl':record.ttl or ttl, \ - type_name:[eval(class_name)(data['value'])] or []} + self.params = None + if not self.record_type == 'CNAME': + self.params = self._params(data, key_name, eval(class_name)) + else: + self.params = {'cname_record': CnameRecord(data['value'])} + self.params['ttl'] = record.ttl + def _params(self, data, key_name, azure_class): + return {key_name: [azure_class(v) for v in data['values']]} \ + if 'values' in data else {key_name: [azure_class(data['value'])]} - class AzureProvider(BaseProvider): ''' Azure DNS Provider - - azure.py: - class: octodns.provider.azure.AzureProvider + + azuredns.py: + class: octodns.provider.azuredns.AzureProvider # Current support of authentication of access to Azure services only # includes using a Service Principal: # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ @@ -71,90 +83,118 @@ class AzureProvider(BaseProvider): # Resource Group name req: resource_group: - testing: test authentication vars located in /home/t-hehwan/vars.txt + TODO: change the config file to use env variables instead of hard-coded keys? + + personal notes: testing: test authentication vars located in /home/t-hehwan/vars.txt ''' - SUPPORTS_GEO = False # TODO. Will add support as project progresses. + SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) - def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): + def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, + *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, client_id=%s, ' 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) credentials = ServicePrincipalCredentials( - client_id = client_id, secret = key, tenant = directory_id + client_id, secret = key, tenant = directory_id ) self._dns_client = DnsManagementClient(credentials, sub_id) self._resource_group = resource_group - - self._azure_zones = None # will be a dictionary. key: name. val: id. - self._azure_records = {} # will be dict by octodns record, az record - - self._supported_types = ['CNAME', 'A', 'AAAA', 'MX', 'SRV', 'NS', 'PTR'] - # TODO: add TXT - - # TODO: health checks a la route53. - - + self._azure_zones = set() - # TODO: add support for all types. First skeleton: add A. - def supports(self, record): - return record._type in self._supported_types - - @property - def azure_zones(self): - if self._azure_zones is None: - self.log.debug('azure_zones: loading') - zones = {} - for zone in self._dns_client.zones.list_by_resource_group(self._resource_group): - zones[zone.name] = zone.id - self._azure_zones = zones - return self._azure_zones - - # Given a zone name, returns the zone id. If DNE, creates it. - def _get_zone_id(self, name, create=False): - self.log.debug('_get_zone_id: name=%s', name) + def _populate_zones(self): + self.log.debug('azure_zones: loading') + for zone in self._dns_client.zones.list_by_resource_group( + self._resource_group): + self._azure_zones.add(zone.name) + + def _check_zone(self, name, create=False): + ''' + Checks whether a zone specified in a source exist in Azure server. + Note that Azure zones omit end '.' eg: contoso.com vs contoso.com. + Returns the name if it exists. + + :param name: Name of a zone to checks + :type name: str + :param create: If True, creates the zone of that name. + :type create: bool + + :type return: str or None + ''' + self.log.debug('_check_zone: name=%s', name) try: - id = self._dns_client.zones.get(self._resource_group, name) - self.log.debug('_get_zone_id: id=%s', id) - return id + if name in self._azure_zones: + return name + if self._dns_client.zones.get(self._resource_group, name): + self._azure_zones.add(name) + return name except: if create: - self.log.debug('_get_zone_id: no matching zone; creating %s', name) - #TODO: write - return None #placeholder - return None + try: + self.log.debug('_check_zone: no matching zone; creating %s', + name) + if self._dns_client.zones.create_or_update( + self._resource_group, name, Zone('global')): #TODO: figure out what location should be + return name + except: + raise + return None - # Create a dictionary of record objects by zone and octodns record names - # TODO: add geo parsing - def populate(self, zone, target): - zone_name = zone.name[0:len(zone.name)-1]#Azure zone names do not include suffix . + def populate(self, zone, target=False): + ''' + Required function of manager.py. + + Special notes for Azure. Azure zone names omit final '.' + Azure record names for '' are represented by '@' + Azure records created through online interface may have null values + (eg, no IP address for A record). Specific quirks such as these are + responsible for any strange parsing. + + :param zone: A dns zone + :type zone: octodns.zone.Zone + :param target: Checks if Azure is source or target of config. + Currently only supports as a target. Does not use. + :type target: bool + + + TODO: azure interface allows null values. If this attempts to populate with them, will fail. add safety check (simply delete records with null values?) + + :type return: void + ''' + zone_name = zone.name[0:len(zone.name)-1] self.log.debug('populate: name=%s', zone_name) before = len(zone.records) - zone_id = self._get_zone_id(zone_name) - if zone_id: - #records = defaultdict(list) - for type in self._supported_types: - # print('populate. type: {}'.format(type),file=sys.stderr) - for azrecord in self._dns_client.record_sets.list_by_type(self._resource_group, zone_name, type): - # print(azrecord, file=sys.stderr) + + self._populate_zones() + if self._check_zone(zone_name): + for typ in self.SUPPORTS: + for azrecord in self._dns_client.record_sets.list_by_type( + self._resource_group, zone_name, typ): record_name = azrecord.name if azrecord.name != '@' else '' - data = self._type_and_ttl(type, azrecord, - getattr(self, '_data_for_{}'.format(type))(azrecord)) # TODO: azure online interface allows None values. must validate. + data = self._type_and_ttl(typ, azrecord.ttl, + getattr(self, '_data_for_{}'.format(typ))(azrecord)) + record = Record.new(zone, record_name, data, source=self) - # print('HERE0',file=sys.stderr) zone.add_record(record) - self._azure_records[record] = _AzureRecord(self._resource_group, record, data) - # print('HERE1',file=sys.stderr) + self.log.info('populate: found %s records', len(zone.records)-before) - - # might not need - def _get_type(azrecord): - azrecord['type'].split('/')[-1] - - def _type_and_ttl(self, type, azrecord, data): - data['type'] = type - data['ttl'] = azrecord.ttl + + def _type_and_ttl(self, typ, ttl, data): + ''' Adds type and ttl fields to return dictionary. + + :param typ: The type of a record + :type typ: str + :param ttl: The ttl of a record + :type ttl: int + :param data: Dictionary holding values of a record. eg, IP addresses + :type data: {'values': [...]} or {'value': [...]} + + :type return: {...} + ''' + data['type'] = typ + data['ttl'] = ttl return data def _data_for_A(self, azrecord): @@ -164,12 +204,10 @@ class AzureProvider(BaseProvider): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} def _data_for_TXT(self, azrecord): - print('azure',file=sys.stderr) - print([ar.value for ar in azrecord.txt_records], file=sys.stderr) - print('',file=sys.stderr) - return {'values': [ar.value for ar in azrecord.txt_records]} + return {'values': \ + [reduce((lambda a,b:a+b), ar.value) for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): + def _data_for_CNAME(self, azrecord): #TODO: see TODO in population comment. try: val = azrecord.cname_record.cname if not val.endswith('.'): @@ -178,7 +216,7 @@ class AzureProvider(BaseProvider): except: return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. - def _data_for_PTR(self, azrecord): + def _data_for_PTR(self, azrecord): #TODO: see TODO in population comment. try: val = azrecord.ptr_records[0].ptdrname if not val.endswith('.'): @@ -198,21 +236,23 @@ class AzureProvider(BaseProvider): 'target': ar.target} for ar in azrecord.srv_records] } - def _data_for_NS(self, azrecord): + def _data_for_NS(self, azrecord): #TODO: see TODO in population comment. def period_validate(string): return string if string.endswith('.') else string + '.' vals = [ar.nsdname for ar in azrecord.ns_records] return {'values': [period_validate(val) for val in vals]} def _apply_Create(self, change): - new = change.new - - #validate that the zone exists. - #self._get_zone_id(new.name, create=True) - - ar = _AzureRecord(self._resource_group, new, new.data) - + ''' A record from change must be created. + + :param change: a change object + :type change: octodns.record.Change + + :type return: void + ''' + ar = _AzureRecord(self._resource_group, change.new) create = self._dns_client.record_sets.create_or_update + create(resource_group_name=ar.resource_group, zone_name=ar.zone_name, relative_record_set_name=ar.relative_record_set_name, @@ -220,56 +260,33 @@ class AzureProvider(BaseProvider): parameters=ar.params) def _apply_Delete(self, change): - existing = change.existing - ar = _AzureRecord(self._resource_group, existing) + ar = _AzureRecord(self._resource_group, change.existing) delete = self._dns_client.record_sets.delete + delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) def _apply_Update(self, change): - self._apply_Delete(change) self._apply_Create(change) - # type plan: Plan class from .base def _apply(self, plan): + ''' + Required function of manager.py + + :param plan: Contains the zones and changes to be made + :type plan: octodns.provider.base.Plan + + :type return: void + ''' desired = plan.desired changes = plan.changes self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - # validate that the zone exists. function creates zone if DNE. - self._get_zone_id(desired.name) + azure_zone_name = desired.name[0:len(desired.name)-1] + self._check_zone(azure_zone_name, create=True) - # Some parsing bits to call _mod_Create or _mod_Delete. - # changes is a list of Delete and Create objects. for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) - - - # ********** - # Figuring out what object plan is. - - # self._executor = ThreadPoolExecutor(max_workers) - # futures.append(self._executor.submit(self._populate_and_plan, - # zone_name, sources, targets)) - # plans = [p for f in futures for p in f.results()] - # type of plans[0] == type of one output of _populate_and_plan - - # for target, plan in plans: - # apply(plan) - - - # type(target) == BaseProvider - # type(plan) == Plan() - - # Plan(existing, desired, changes) - # existing.type == desired.type == Zone(desired.name, desired.sub_zones) - # Zone(name, sub_zones) (str and set of strs) - # changes.type = [Delete/Create] - - - # Starts with sync in main() of sync. - # {u'values': ['3.3.3.3', '4.4.4.4'], u'type': 'A', u'ttl': 3600} - # {u'type': u'A', u'value': [u'3.3.3.3', u'4.4.4.4'], u'ttl': 3600L} \ No newline at end of file From cc47bd70348ac711da8dc6d3ef2ce4b74a2642b4 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Tue, 27 Jun 2017 12:10:57 -0700 Subject: [PATCH 09/24] Fixed bug for MX and SRV. Added Azure test suite as well. --- octodns/provider/azuredns.py | 287 ++++++++++++++---------- rb.txt | 8 - tests/test_octodns_provider_azuredns.py | 171 ++++++++++++++ 3 files changed, 342 insertions(+), 124 deletions(-) delete mode 100755 rb.txt create mode 100644 tests/test_octodns_provider_azuredns.py diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 07b39fe..013613a 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import sys from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient @@ -14,53 +13,120 @@ from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ from functools import reduce import logging -import re -from ..record import Record, Update +from ..record import Record from .base import BaseProvider + class _AzureRecord(object): - ''' - Wrapper for OctoDNS record. - azuredns.py: + ''' Wrapper for OctoDNS record. + azuredns.py: class: octodns.provider.azuredns._AzureRecord - An _AzureRecord is easily accessible to the Azure DNS Management library - functions and is used to wrap all relevant data to create a record in + An _AzureRecord is easily accessible to Azure DNS Management library + functions and is used to wrap all relevant data to create a record in Azure. ''' - + def __init__(self, resource_group, record, values=None): - ''' + ''' :param resource_group: The name of resource group in Azure - :type resource_group: str - :param record: An OctoDNS record + :type resource_group: str + :param record: An OctoDNS record :type record: ..record.Record :param values: Parameters for a record. eg IP address, port, domain name, etc. Values usually read from record.data :type values: {'values': [...]} or {'value': [...]} - + :type return: _AzureRecord ''' self.resource_group = resource_group - self.zone_name = record.zone.name[0:len(record.zone.name)-1] + self.zone_name = record.zone.name[0:len(record.zone.name) - 1] self.relative_record_set_name = record.name or '@' self.record_type = record._type - + data = values or record.data format_u_s = '' if record._type == 'A' else '_' - key_name ='{}{}records'.format(self.record_type, format_u_s).lower() + key_name = '{}{}records'.format(self.record_type, format_u_s).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) - self.params = None - if not self.record_type == 'CNAME': - self.params = self._params(data, key_name, eval(class_name)) - else: - self.params = {'cname_record': CnameRecord(data['value'])} + + self.params = getattr(self, '_params_for_{}'.format(record._type)) + self.params = self.params(data, key_name, eval(class_name)) self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): return {key_name: [azure_class(v) for v in data['values']]} \ if 'values' in data else {key_name: [azure_class(data['value'])]} + _params_for_A = _params + _params_for_AAAA = _params + _params_for_NS = _params + _params_for_PTR = _params + _params_for_TXT = _params + + def _params_for_SRV(self, data, key_name, azure_class): + params = [] + if 'values' in data: + for vals in data['values']: + params.append(azure_class(vals['priority'], + vals['weight'], + vals['port'], + vals['target'])) + else: + params.append(azure_class(data['value']['priority'], + data['value']['weight'], + data['value']['port'], + data['value']['target'])) + return {key_name: params} + + def _params_for_MX(self, data, key_name, azure_class): + params = [] + if 'values' in data: + for vals in data['values']: + params.append(azure_class(vals['priority'], + vals['value'])) + else: + params.append(azure_class(data['value']['priority'], + data['value']['value'])) + return {key_name: params} + + def _params_for_CNAME(self, data, key_name, azure_class): + return {'cname_record': CnameRecord(data['value'])} + + def _equals(self, b): + def parse_dict(params): + vals = [] + for char in params: + if char != 'ttl': + list_records = params[char] + try: + for record in list_records: + vals.append(record.__dict__) + except: + vals.append(list_records.__dict__) + vals.sort() + return vals + + return (self.resource_group == b.resource_group) & \ + (self.zone_name == b.zone_name) & \ + (self.record_type == b.record_type) & \ + (self.params['ttl'] == b.params['ttl']) & \ + (parse_dict(self.params) == parse_dict(b.params)) & \ + (self.relative_record_set_name == b.relative_record_set_name) + + def __str__(self): + string = 'Zone: {}; '.format(self.zone_name) + string += 'Name: {}; '.format(self.relative_record_set_name) + string += 'Type: {}; '.format(self.record_type) + string += 'Ttl: {}; '.format(self.params['ttl']) + for char in self.params: + if char != 'ttl': + try: + for rec in self.params[char]: + string += 'Record: {}; '.format(rec.__dict__) + except: + string += 'Record: {}; '.format(self.params[char].__dict__) + return string + class AzureProvider(BaseProvider): ''' @@ -68,11 +134,11 @@ class AzureProvider(BaseProvider): azuredns.py: class: octodns.provider.azuredns.AzureProvider - # Current support of authentication of access to Azure services only - # includes using a Service Principal: + # Current support of authentication of access to Azure services only + # includes using a Service Principal: # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ # resource-group-create-service-principal-portal - # The Azure Active Directory Application ID (referred to client ID) req: + # The Azure Active Directory Application ID (aka client ID) req: client_id: # Authentication Key Value req: key: @@ -82,32 +148,33 @@ class AzureProvider(BaseProvider): sub_id: # Resource Group name req: resource_group: - - TODO: change the config file to use env variables instead of hard-coded keys? - - personal notes: testing: test authentication vars located in /home/t-hehwan/vars.txt + + TODO: change config file to use env vars instead of hard-coded keys + + personal notes: testing: test authentication vars located in + /home/t-hehwan/vars.txt ''' SUPPORTS_GEO = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) - - def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, - *args, **kwargs): + + def __init__(self, id, client_id, key, directory_id, sub_id, + resource_group, *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, client_id=%s, ' - 'key=***, directory_id:%s', id, client_id, directory_id) + 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) credentials = ServicePrincipalCredentials( - client_id, secret = key, tenant = directory_id + client_id, secret=key, tenant=directory_id ) self._dns_client = DnsManagementClient(credentials, sub_id) self._resource_group = resource_group self._azure_zones = set() - + def _populate_zones(self): self.log.debug('azure_zones: loading') - for zone in self._dns_client.zones.list_by_resource_group( - self._resource_group): + list_zones = self._dns_client.zones.list_by_resource_group + for zone in list_zones(self._resource_group): self._azure_zones.add(zone.name) def _check_zone(self, name, create=False): @@ -115,12 +182,12 @@ class AzureProvider(BaseProvider): Checks whether a zone specified in a source exist in Azure server. Note that Azure zones omit end '.' eg: contoso.com vs contoso.com. Returns the name if it exists. - + :param name: Name of a zone to checks :type name: str :param create: If True, creates the zone of that name. :type create: bool - + :type return: str or None ''' self.log.debug('_check_zone: name=%s', name) @@ -130,163 +197,151 @@ class AzureProvider(BaseProvider): if self._dns_client.zones.get(self._resource_group, name): self._azure_zones.add(name) return name - except: + except: # TODO: figure out what location should be if create: try: - self.log.debug('_check_zone: no matching zone; creating %s', - name) - if self._dns_client.zones.create_or_update( - self._resource_group, name, Zone('global')): #TODO: figure out what location should be + self.log.debug('_check_zone:no matching zone; creating %s', + name) + create_zone = self._dns_client.zones.create_or_update + if create_zone(self._resource_group, name, Zone('global')): return name except: raise return None - - def populate(self, zone, target=False): + + def populate(self, zone, target=False, lenient=False): ''' Required function of manager.py. - + Special notes for Azure. Azure zone names omit final '.' Azure record names for '' are represented by '@' Azure records created through online interface may have null values (eg, no IP address for A record). Specific quirks such as these are responsible for any strange parsing. - + :param zone: A dns zone :type zone: octodns.zone.Zone - :param target: Checks if Azure is source or target of config. + :param target: Checks if Azure is source or target of config. Currently only supports as a target. Does not use. :type target: bool - - - TODO: azure interface allows null values. If this attempts to populate with them, will fail. add safety check (simply delete records with null values?) - + + + TODO: azure interface allows null values. If this attempts to + populate with them, will fail. add safety check (simply delete + records with null values?) + :type return: void ''' - zone_name = zone.name[0:len(zone.name)-1] + zone_name = zone.name[0:len(zone.name) - 1] self.log.debug('populate: name=%s', zone_name) before = len(zone.records) self._populate_zones() if self._check_zone(zone_name): for typ in self.SUPPORTS: - for azrecord in self._dns_client.record_sets.list_by_type( - self._resource_group, zone_name, typ): + records = self._dns_client.record_sets.list_by_type + for azrecord in records(self._resource_group, zone_name, typ): record_name = azrecord.name if azrecord.name != '@' else '' - data = self._type_and_ttl(typ, azrecord.ttl, - getattr(self, '_data_for_{}'.format(typ))(azrecord)) - + data = getattr(self, '_data_for_{}'.format(typ))(azrecord) + data['type'] = typ + data['ttl'] = azrecord.ttl + record = Record.new(zone, record_name, data, source=self) zone.add_record(record) - self.log.info('populate: found %s records', len(zone.records)-before) + self.log.info('populate: found %s records', len(zone.records) - before) - def _type_and_ttl(self, typ, ttl, data): - ''' Adds type and ttl fields to return dictionary. - - :param typ: The type of a record - :type typ: str - :param ttl: The ttl of a record - :type ttl: int - :param data: Dictionary holding values of a record. eg, IP addresses - :type data: {'values': [...]} or {'value': [...]} - - :type return: {...} - ''' - data['type'] = typ - data['ttl'] = ttl - return data - def _data_for_A(self, azrecord): return {'values': [ar.ipv4_address for ar in azrecord.arecords]} - + def _data_for_AAAA(self, azrecord): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} - - def _data_for_TXT(self, azrecord): - return {'values': \ - [reduce((lambda a,b:a+b), ar.value) for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): #TODO: see TODO in population comment. + def _data_for_TXT(self, azrecord): + return {'values': [reduce((lambda a, b: a + b), ar.value) + for ar in azrecord.txt_records]} + + def _data_for_CNAME(self, azrecord): # TODO: see TODO in pop comment. try: val = azrecord.cname_record.cname if not val.endswith('.'): val += '.' return {'value': val} except: - return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. - - def _data_for_PTR(self, azrecord): #TODO: see TODO in population comment. + return {'value': '.'} # TODO: this is a bad fix. but octo checks + # that cnames have trailing '.' while azure allows creating cnames + # on the online interface with no value. + + def _data_for_PTR(self, azrecord): # TODO: see TODO in population comment. try: val = azrecord.ptr_records[0].ptdrname if not val.endswith('.'): val += '.' return {'value': val} except: - return {'value': '.' } #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. - + return {'value': '.'} + def _data_for_MX(self, azrecord): - return {'values': [{'priority':ar.preference, - 'value':ar.exchange} for ar in azrecord.mx_records]} - + return {'values': [{'priority': ar.preference, 'value': ar.exchange} + for ar in azrecord.mx_records] + } + def _data_for_SRV(self, azrecord): - return {'values': [{'priority': ar.priority, - 'weight': ar.weight, - 'port': ar.port, - 'target': ar.target} for ar in azrecord.srv_records] - } - - def _data_for_NS(self, azrecord): #TODO: see TODO in population comment. + return {'values': [{'priority': ar.priority, 'weight': ar.weight, + 'port': ar.port, 'target': ar.target} + for ar in azrecord.srv_records] + } + + def _data_for_NS(self, azrecord): # TODO: see TODO in population comment. def period_validate(string): return string if string.endswith('.') else string + '.' vals = [ar.nsdname for ar in azrecord.ns_records] return {'values': [period_validate(val) for val in vals]} def _apply_Create(self, change): - ''' A record from change must be created. - + '''A record from change must be created. + :param change: a change object :type change: octodns.record.Change - + :type return: void ''' ar = _AzureRecord(self._resource_group, change.new) create = self._dns_client.record_sets.create_or_update - - create(resource_group_name=ar.resource_group, - zone_name=ar.zone_name, - relative_record_set_name=ar.relative_record_set_name, - record_type=ar.record_type, + + create(resource_group_name=ar.resource_group, + zone_name=ar.zone_name, + relative_record_set_name=ar.relative_record_set_name, + record_type=ar.record_type, parameters=ar.params) - + def _apply_Delete(self, change): ar = _AzureRecord(self._resource_group, change.existing) delete = self._dns_client.record_sets.delete - - delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, + + delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - + def _apply_Update(self, change): self._apply_Create(change) - + def _apply(self, plan): ''' Required function of manager.py - + :param plan: Contains the zones and changes to be made :type plan: octodns.provider.base.Plan - + :type return: void ''' desired = plan.desired changes = plan.changes - self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - - azure_zone_name = desired.name[0:len(desired.name)-1] + + azure_zone_name = desired.name[0:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) - + for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) - diff --git a/rb.txt b/rb.txt deleted file mode 100755 index d6a7949..0000000 --- a/rb.txt +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -#script to rebuild octodns quickly - -sudo rm -r /home/t-hehwan/GitHub/octodns/build -sudo rm -r /home/t-hehwan/GitHub/octodns/octodns.egg-info -sudo python /home/t-hehwan/GitHub/octodns/setup.py -q build -sudo python /home/t-hehwan/GitHub/octodns/setup.py -q install -octodns-sync --config-file=./config/production.yaml \ No newline at end of file diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py new file mode 100644 index 0000000..db4d293 --- /dev/null +++ b/tests/test_octodns_provider_azuredns.py @@ -0,0 +1,171 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from octodns.record import Create, Delete, Record, Update +from octodns.provider.azuredns import _AzureRecord, AzureProvider +from octodns.zone import Zone + +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone as AzureZone + +from octodns.zone import Zone + +from unittest import TestCase +import sys + + +class Test_AzureRecord(TestCase): + zone = Zone(name='unit.tests.', sub_zones=[]) + octo_records = [] + octo_records.append(Record.new(zone, '', { + 'ttl': 0, + 'type': 'A', + 'values': ['1.2.3.4', '10.10.10.10'] + })) + octo_records.append(Record.new(zone, 'a', { + 'ttl': 1, + 'type': 'A', + 'values': ['1.2.3.4', '1.1.1.1'], + })) + octo_records.append(Record.new(zone, 'aa', { + 'ttl': 9001, + 'type': 'A', + 'values': ['1.2.4.3'] + })) + octo_records.append(Record.new(zone, 'aaa', { + 'ttl': 2, + 'type': 'A', + 'values': ['1.1.1.3'] + })) + octo_records.append(Record.new(zone, 'cname', { + 'ttl': 3, + 'type': 'CNAME', + 'value': 'a.unit.tests.', + })) + octo_records.append(Record.new(zone, '', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }, { + 'priority': 20, + 'value': 'mx2.unit.tests.', + }] + })) + octo_records.append(Record.new(zone, '', { + 'ttl': 4, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.'], + })) + octo_records.append(Record.new(zone, '', { + 'ttl': 5, + 'type': 'NS', + 'value': 'ns1.unit.tests.', + })) + octo_records.append(Record.new(zone, '_srv._tcp', { + 'ttl': 6, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 12, + 'weight': 30, + 'port': 30, + 'target': 'foo-2.unit.tests.', + }] + })) + + azure_records = [] + _base0 = _AzureRecord('TestAzure', octo_records[0]) + _base0.zone_name = 'unit.tests' + _base0.relative_record_set_name = '@' + _base0.record_type = 'A' + _base0.params['ttl'] = 0 + _base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')] + azure_records.append(_base0) + + _base1 = _AzureRecord('TestAzure', octo_records[1]) + _base1.zone_name = 'unit.tests' + _base1.relative_record_set_name = 'a' + _base1.record_type = 'A' + _base1.params['ttl'] = 1 + _base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')] + azure_records.append(_base1) + + _base2 = _AzureRecord('TestAzure', octo_records[2]) + _base2.zone_name = 'unit.tests' + _base2.relative_record_set_name = 'aa' + _base2.record_type = 'A' + _base2.params['ttl'] = 9001 + _base2.params['arecords'] = ARecord('1.2.4.3') + azure_records.append(_base2) + + _base3 = _AzureRecord('TestAzure', octo_records[3]) + _base3.zone_name = 'unit.tests' + _base3.relative_record_set_name = 'aaa' + _base3.record_type = 'A' + _base3.params['ttl'] = 2 + _base3.params['arecords'] = ARecord('1.1.1.3') + azure_records.append(_base3) + + _base4 = _AzureRecord('TestAzure', octo_records[4]) + _base4.zone_name = 'unit.tests' + _base4.relative_record_set_name = 'cname' + _base4.record_type = 'CNAME' + _base4.params['ttl'] = 3 + _base4.params['cname_record'] = CnameRecord('a.unit.tests.') + azure_records.append(_base4) + + _base5 = _AzureRecord('TestAzure', octo_records[5]) + _base5.zone_name = 'unit.tests' + _base5.relative_record_set_name = '@' + _base5.record_type = 'MX' + _base5.params['ttl'] = 3 + _base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'), + MxRecord(20, 'mx2.unit.tests.')] + azure_records.append(_base5) + + _base6 = _AzureRecord('TestAzure', octo_records[6]) + _base6.zone_name = 'unit.tests' + _base6.relative_record_set_name = '@' + _base6.record_type = 'NS' + _base6.params['ttl'] = 4 + _base6.params['ns_records'] = [NsRecord('ns1.unit.tests.'), + NsRecord('ns2.unit.tests.')] + azure_records.append(_base6) + + _base7 = _AzureRecord('TestAzure', octo_records[7]) + _base7.zone_name = 'unit.tests' + _base7.relative_record_set_name = '@' + _base7.record_type = 'NS' + _base7.params['ttl'] = 5 + _base7.params['ns_records'] = [NsRecord('ns1.unit.tests.')] + azure_records.append(_base7) + + _base8 = _AzureRecord('TestAzure', octo_records[8]) + _base8.zone_name = 'unit.tests' + _base8.relative_record_set_name = '_srv._tcp' + _base8.record_type = 'SRV' + _base8.params['ttl'] = 6 + _base8.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'), + SrvRecord(12, 30, 30, 'foo-2.unit.tests.')] + azure_records.append(_base8) + + def test_azure_record(self): + assert(len(self.azure_records) == len(self.octo_records)) + for i in range(len(self.azure_records)): + octo = _AzureRecord('TestAzure', self.octo_records[i]) + assert(self.azure_records[i]._equals(octo)) + + +class TestAzureDnsProvider(TestCase): + def test_populate(self): + pass # placeholder From 08d3fda99ef17e4ae0ca529989c14a356b8f925c Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Wed, 28 Jun 2017 14:00:59 -0700 Subject: [PATCH 10/24] safety check for azure null values --- octodns/provider/azuredns.py | 108 ++++++++++++++++++++--------------- 1 file changed, 63 insertions(+), 45 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 013613a..6a2db41 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -11,7 +11,7 @@ from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone from functools import reduce - +import sys import logging from ..record import Record from .base import BaseProvider @@ -26,7 +26,7 @@ class _AzureRecord(object): Azure. ''' - def __init__(self, resource_group, record, values=None): + def __init__(self, resource_group, record, delete=False): ''' :param resource_group: The name of resource group in Azure :type resource_group: str @@ -43,14 +43,16 @@ class _AzureRecord(object): self.relative_record_set_name = record.name or '@' self.record_type = record._type - data = values or record.data + if delete: + return + format_u_s = '' if record._type == 'A' else '_' key_name = '{}{}records'.format(self.record_type, format_u_s).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) self.params = getattr(self, '_params_for_{}'.format(record._type)) - self.params = self.params(data, key_name, eval(class_name)) + self.params = self.params(record.data, key_name, eval(class_name)) self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): @@ -71,7 +73,7 @@ class _AzureRecord(object): vals['weight'], vals['port'], vals['target'])) - else: + else: # single value at key 'value' params.append(azure_class(data['value']['priority'], data['value']['weight'], data['value']['port'], @@ -82,17 +84,23 @@ class _AzureRecord(object): params = [] if 'values' in data: for vals in data['values']: - params.append(azure_class(vals['priority'], - vals['value'])) - else: - params.append(azure_class(data['value']['priority'], - data['value']['value'])) + params.append(azure_class(vals['preference'], + vals['exchange'])) + else: # single value at key 'value' + params.append(azure_class(data['value']['preference'], + data['value']['exchange'])) return {key_name: params} def _params_for_CNAME(self, data, key_name, azure_class): return {'cname_record': CnameRecord(data['value'])} def _equals(self, b): + '''Checks whether two records are equal by comparing all fields. + :param b: Another _AzureRecord object + :type b: _AzureRecord + + :type return: bool + ''' def parse_dict(params): vals = [] for char in params: @@ -114,6 +122,9 @@ class _AzureRecord(object): (self.relative_record_set_name == b.relative_record_set_name) def __str__(self): + '''String representation of an _AzureRecord. + :type return: str + ''' string = 'Zone: {}; '.format(self.zone_name) string += 'Name: {}; '.format(self.relative_record_set_name) string += 'Type: {}; '.format(self.record_type) @@ -128,6 +139,10 @@ class _AzureRecord(object): return string +def period_validate(string): + return string if string.endswith('.') else string + '.' + + class AzureProvider(BaseProvider): ''' Azure DNS Provider @@ -213,22 +228,24 @@ class AzureProvider(BaseProvider): ''' Required function of manager.py. - Special notes for Azure. Azure zone names omit final '.' - Azure record names for '' are represented by '@' + Special notes for Azure. + Azure zone names omit final '.' + Azure root records names are represented by '@'. OctoDNS uses '' Azure records created through online interface may have null values - (eg, no IP address for A record). Specific quirks such as these are - responsible for any strange parsing. + (eg, no IP address for A record). + Azure online interface allows constructing records with null values + which are destroyed by _apply. + + Specific quirks such as these are responsible for any non-obvious + parsing in this function and the functions '_params_for_*'. :param zone: A dns zone :type zone: octodns.zone.Zone :param target: Checks if Azure is source or target of config. Currently only supports as a target. Does not use. :type target: bool - - - TODO: azure interface allows null values. If this attempts to - populate with them, will fail. add safety check (simply delete - records with null values?) + :param lenient: Unused. Check octodns.manager for usage. + :type lenient: bool :type return: void ''' @@ -261,40 +278,38 @@ class AzureProvider(BaseProvider): return {'values': [reduce((lambda a, b: a + b), ar.value) for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): # TODO: see TODO in pop comment. - try: - val = azrecord.cname_record.cname - if not val.endswith('.'): - val += '.' - return {'value': val} - except: - return {'value': '.'} # TODO: this is a bad fix. but octo checks - # that cnames have trailing '.' while azure allows creating cnames - # on the online interface with no value. + def _data_for_CNAME(self, azrecord): + '''Parsing data from Azure DNS Client record call + :param azrecord: a return of a call to list azure records + :type azrecord: azure.mgmt.dns.models.RecordSet - def _data_for_PTR(self, azrecord): # TODO: see TODO in population comment. + :type return: dict + + CNAME and PTR both use the catch block to catch possible empty + records. Refer to population comment. + ''' try: - val = azrecord.ptr_records[0].ptdrname - if not val.endswith('.'): - val += '.' - return {'value': val} + return {'value': period_validate(azrecord.cname_record.cname)} + except: + return {'value': '.'} + + def _data_for_PTR(self, azrecord): + try: + return {'value': period_validate(azrecord.ptr_records[0].ptdrname)} except: return {'value': '.'} def _data_for_MX(self, azrecord): - return {'values': [{'priority': ar.preference, 'value': ar.exchange} - for ar in azrecord.mx_records] - } + return {'values': [{'preference': ar.preference, + 'exchange': ar.exchange} + for ar in azrecord.mx_records]} def _data_for_SRV(self, azrecord): return {'values': [{'priority': ar.priority, 'weight': ar.weight, 'port': ar.port, 'target': ar.target} - for ar in azrecord.srv_records] - } + for ar in azrecord.srv_records]} - def _data_for_NS(self, azrecord): # TODO: see TODO in population comment. - def period_validate(string): - return string if string.endswith('.') else string + '.' + def _data_for_NS(self, azrecord): vals = [ar.nsdname for ar in azrecord.ns_records] return {'values': [period_validate(val) for val in vals]} @@ -315,15 +330,18 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) + print('* Success Create/Update: {}'.format(ar), file=sys.stderr) + + _apply_Update = _apply_Create + def _apply_Delete(self, change): - ar = _AzureRecord(self._resource_group, change.existing) + ar = _AzureRecord(self._resource_group, change.existing, delete=True) delete = self._dns_client.record_sets.delete delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - def _apply_Update(self, change): - self._apply_Create(change) + print('* Success Delete: {}'.format(ar), file=sys.stderr) def _apply(self, plan): ''' From 0b2275c4e6388d25a28e501152fae4a88f2cf0a1 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 30 Jun 2017 11:06:42 -0700 Subject: [PATCH 11/24] Added complete error testing suite for azuredns --- octodns/provider/azuredns.py | 62 ++-- tests/test_octodns_provider_azuredns.py | 456 ++++++++++++++++-------- 2 files changed, 338 insertions(+), 180 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 6a2db41..cf0e2d5 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -128,6 +128,8 @@ class _AzureRecord(object): string = 'Zone: {}; '.format(self.zone_name) string += 'Name: {}; '.format(self.relative_record_set_name) string += 'Type: {}; '.format(self.record_type) + if not hasattr(self, 'params'): + return string string += 'Ttl: {}; '.format(self.params['ttl']) for char in self.params: if char != 'ttl': @@ -139,7 +141,7 @@ class _AzureRecord(object): return string -def period_validate(string): +def _validate_per(string): return string if string.endswith('.') else string + '.' @@ -209,20 +211,18 @@ class AzureProvider(BaseProvider): try: if name in self._azure_zones: return name - if self._dns_client.zones.get(self._resource_group, name): - self._azure_zones.add(name) - return name - except: # TODO: figure out what location should be + self._dns_client.zones.get(self._resource_group, name) + self._azure_zones.add(name) + return name + except: if create: - try: - self.log.debug('_check_zone:no matching zone; creating %s', - name) - create_zone = self._dns_client.zones.create_or_update - if create_zone(self._resource_group, name, Zone('global')): - return name - except: - raise - return None + self.log.debug('_check_zone:no matching zone; creating %s', + name) + create_zone = self._dns_client.zones.create_or_update + create_zone(self._resource_group, name, Zone('global')) + return name + else: + raise def populate(self, zone, target=False, lenient=False): ''' @@ -254,17 +254,21 @@ class AzureProvider(BaseProvider): before = len(zone.records) self._populate_zones() - if self._check_zone(zone_name): - for typ in self.SUPPORTS: - records = self._dns_client.record_sets.list_by_type - for azrecord in records(self._resource_group, zone_name, typ): - record_name = azrecord.name if azrecord.name != '@' else '' - data = getattr(self, '_data_for_{}'.format(typ))(azrecord) - data['type'] = typ - data['ttl'] = azrecord.ttl + self._check_zone(zone_name) - record = Record.new(zone, record_name, data, source=self) - zone.add_record(record) + _records = set() + records = self._dns_client.record_sets.list_by_dns_zone + for azrecord in records(self._resource_group, zone_name): + if azrecord.type in self.SUPPORTS: + _records.add(azrecord) + for azrecord in _records: + record_name = azrecord.name if azrecord.name != '@' else '' + data = getattr(self, '_data_for_{}'.format(azrecord.type)) + data = data(azrecord) + data['type'] = azrecord.type + data['ttl'] = azrecord.ttl + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) self.log.info('populate: found %s records', len(zone.records) - before) @@ -289,13 +293,13 @@ class AzureProvider(BaseProvider): records. Refer to population comment. ''' try: - return {'value': period_validate(azrecord.cname_record.cname)} + return {'value': _validate_per(azrecord.cname_record.cname)} except: return {'value': '.'} def _data_for_PTR(self, azrecord): try: - return {'value': period_validate(azrecord.ptr_records[0].ptdrname)} + return {'value': _validate_per(azrecord.ptr_records[0].ptdrname)} except: return {'value': '.'} @@ -311,7 +315,7 @@ class AzureProvider(BaseProvider): def _data_for_NS(self, azrecord): vals = [ar.nsdname for ar in azrecord.ns_records] - return {'values': [period_validate(val) for val in vals]} + return {'values': [_validate_per(val) for val in vals]} def _apply_Create(self, change): '''A record from change must be created. @@ -330,7 +334,7 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - print('* Success Create/Update: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Create/Update: {}'.format(ar)) _apply_Update = _apply_Create @@ -341,7 +345,7 @@ class AzureProvider(BaseProvider): delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - print('* Success Delete: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): ''' diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index db4d293..edb2db2 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -5,167 +5,321 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from octodns.record import Create, Delete, Record, Update -from octodns.provider.azuredns import _AzureRecord, AzureProvider +from octodns.record import Create, Delete, Record +from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ + _validate_per from octodns.zone import Zone +from octodns.provider.base import Plan from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ - SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone as AzureZone + SrvRecord, NsRecord, PtrRecord, TxtRecord, RecordSet, SoaRecord, \ + Zone as AzureZone +from msrestazure.azure_exceptions import CloudError -from octodns.zone import Zone from unittest import TestCase -import sys +from mock import Mock, patch + + +zone = Zone(name='unit.tests.', sub_zones=[]) +octo_records = [] +octo_records.append(Record.new(zone, '', { + 'ttl': 0, + 'type': 'A', + 'values': ['1.2.3.4', '10.10.10.10']})) +octo_records.append(Record.new(zone, 'a', { + 'ttl': 1, + 'type': 'A', + 'values': ['1.2.3.4', '1.1.1.1']})) +octo_records.append(Record.new(zone, 'aa', { + 'ttl': 9001, + 'type': 'A', + 'values': ['1.2.4.3']})) +octo_records.append(Record.new(zone, 'aaa', { + 'ttl': 2, + 'type': 'A', + 'values': ['1.1.1.3']})) +octo_records.append(Record.new(zone, 'cname', { + 'ttl': 3, + 'type': 'CNAME', + 'value': 'a.unit.tests.'})) +octo_records.append(Record.new(zone, 'mx1', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }, { + 'priority': 20, + 'value': 'mx2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, 'mx2', { + 'ttl': 3, + 'type': 'MX', + 'values': [{ + 'priority': 10, + 'value': 'mx1.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '', { + 'ttl': 4, + 'type': 'NS', + 'values': ['ns1.unit.tests.', 'ns2.unit.tests.']})) +octo_records.append(Record.new(zone, 'foo', { + 'ttl': 5, + 'type': 'NS', + 'value': 'ns1.unit.tests.'})) +octo_records.append(Record.new(zone, '_srv._tcp', { + 'ttl': 6, + 'type': 'SRV', + 'values': [{ + 'priority': 10, + 'weight': 20, + 'port': 30, + 'target': 'foo-1.unit.tests.', + }, { + 'priority': 12, + 'weight': 30, + 'port': 30, + 'target': 'foo-2.unit.tests.', + }]})) +octo_records.append(Record.new(zone, '_srv2._tcp', { + 'ttl': 7, + 'type': 'SRV', + 'values': [{ + 'priority': 12, + 'weight': 17, + 'port': 1, + 'target': 'srvfoo.unit.tests.', + }]})) + +azure_records = [] +_base0 = _AzureRecord('TestAzure', octo_records[0]) +_base0.zone_name = 'unit.tests' +_base0.relative_record_set_name = '@' +_base0.record_type = 'A' +_base0.params['ttl'] = 0 +_base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')] +azure_records.append(_base0) + +_base1 = _AzureRecord('TestAzure', octo_records[1]) +_base1.zone_name = 'unit.tests' +_base1.relative_record_set_name = 'a' +_base1.record_type = 'A' +_base1.params['ttl'] = 1 +_base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')] +azure_records.append(_base1) + +_base2 = _AzureRecord('TestAzure', octo_records[2]) +_base2.zone_name = 'unit.tests' +_base2.relative_record_set_name = 'aa' +_base2.record_type = 'A' +_base2.params['ttl'] = 9001 +_base2.params['arecords'] = ARecord('1.2.4.3') +azure_records.append(_base2) + +_base3 = _AzureRecord('TestAzure', octo_records[3]) +_base3.zone_name = 'unit.tests' +_base3.relative_record_set_name = 'aaa' +_base3.record_type = 'A' +_base3.params['ttl'] = 2 +_base3.params['arecords'] = ARecord('1.1.1.3') +azure_records.append(_base3) + +_base4 = _AzureRecord('TestAzure', octo_records[4]) +_base4.zone_name = 'unit.tests' +_base4.relative_record_set_name = 'cname' +_base4.record_type = 'CNAME' +_base4.params['ttl'] = 3 +_base4.params['cname_record'] = CnameRecord('a.unit.tests.') +azure_records.append(_base4) + +_base5 = _AzureRecord('TestAzure', octo_records[5]) +_base5.zone_name = 'unit.tests' +_base5.relative_record_set_name = 'mx1' +_base5.record_type = 'MX' +_base5.params['ttl'] = 3 +_base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'), + MxRecord(20, 'mx2.unit.tests.')] +azure_records.append(_base5) + +_base6 = _AzureRecord('TestAzure', octo_records[6]) +_base6.zone_name = 'unit.tests' +_base6.relative_record_set_name = 'mx2' +_base6.record_type = 'MX' +_base6.params['ttl'] = 3 +_base6.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.')] +azure_records.append(_base6) + +_base7 = _AzureRecord('TestAzure', octo_records[7]) +_base7.zone_name = 'unit.tests' +_base7.relative_record_set_name = '@' +_base7.record_type = 'NS' +_base7.params['ttl'] = 4 +_base7.params['ns_records'] = [NsRecord('ns1.unit.tests.'), + NsRecord('ns2.unit.tests.')] +azure_records.append(_base7) + +_base8 = _AzureRecord('TestAzure', octo_records[8]) +_base8.zone_name = 'unit.tests' +_base8.relative_record_set_name = 'foo' +_base8.record_type = 'NS' +_base8.params['ttl'] = 5 +_base8.params['ns_records'] = [NsRecord('ns1.unit.tests.')] +azure_records.append(_base8) + +_base9 = _AzureRecord('TestAzure', octo_records[9]) +_base9.zone_name = 'unit.tests' +_base9.relative_record_set_name = '_srv._tcp' +_base9.record_type = 'SRV' +_base9.params['ttl'] = 6 +_base9.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'), + SrvRecord(12, 30, 30, 'foo-2.unit.tests.')] +azure_records.append(_base9) + +_base10 = _AzureRecord('TestAzure', octo_records[10]) +_base10.zone_name = 'unit.tests' +_base10.relative_record_set_name = '_srv2._tcp' +_base10.record_type = 'SRV' +_base10.params['ttl'] = 7 +_base10.params['srv_records'] = [SrvRecord(12, 17, 1, 'srvfoo.unit.tests.')] +azure_records.append(_base10) class Test_AzureRecord(TestCase): - zone = Zone(name='unit.tests.', sub_zones=[]) - octo_records = [] - octo_records.append(Record.new(zone, '', { - 'ttl': 0, - 'type': 'A', - 'values': ['1.2.3.4', '10.10.10.10'] - })) - octo_records.append(Record.new(zone, 'a', { - 'ttl': 1, - 'type': 'A', - 'values': ['1.2.3.4', '1.1.1.1'], - })) - octo_records.append(Record.new(zone, 'aa', { - 'ttl': 9001, - 'type': 'A', - 'values': ['1.2.4.3'] - })) - octo_records.append(Record.new(zone, 'aaa', { - 'ttl': 2, - 'type': 'A', - 'values': ['1.1.1.3'] - })) - octo_records.append(Record.new(zone, 'cname', { - 'ttl': 3, - 'type': 'CNAME', - 'value': 'a.unit.tests.', - })) - octo_records.append(Record.new(zone, '', { - 'ttl': 3, - 'type': 'MX', - 'values': [{ - 'priority': 10, - 'value': 'mx1.unit.tests.', - }, { - 'priority': 20, - 'value': 'mx2.unit.tests.', - }] - })) - octo_records.append(Record.new(zone, '', { - 'ttl': 4, - 'type': 'NS', - 'values': ['ns1.unit.tests.', 'ns2.unit.tests.'], - })) - octo_records.append(Record.new(zone, '', { - 'ttl': 5, - 'type': 'NS', - 'value': 'ns1.unit.tests.', - })) - octo_records.append(Record.new(zone, '_srv._tcp', { - 'ttl': 6, - 'type': 'SRV', - 'values': [{ - 'priority': 10, - 'weight': 20, - 'port': 30, - 'target': 'foo-1.unit.tests.', - }, { - 'priority': 12, - 'weight': 30, - 'port': 30, - 'target': 'foo-2.unit.tests.', - }] - })) - - azure_records = [] - _base0 = _AzureRecord('TestAzure', octo_records[0]) - _base0.zone_name = 'unit.tests' - _base0.relative_record_set_name = '@' - _base0.record_type = 'A' - _base0.params['ttl'] = 0 - _base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')] - azure_records.append(_base0) - - _base1 = _AzureRecord('TestAzure', octo_records[1]) - _base1.zone_name = 'unit.tests' - _base1.relative_record_set_name = 'a' - _base1.record_type = 'A' - _base1.params['ttl'] = 1 - _base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')] - azure_records.append(_base1) - - _base2 = _AzureRecord('TestAzure', octo_records[2]) - _base2.zone_name = 'unit.tests' - _base2.relative_record_set_name = 'aa' - _base2.record_type = 'A' - _base2.params['ttl'] = 9001 - _base2.params['arecords'] = ARecord('1.2.4.3') - azure_records.append(_base2) - - _base3 = _AzureRecord('TestAzure', octo_records[3]) - _base3.zone_name = 'unit.tests' - _base3.relative_record_set_name = 'aaa' - _base3.record_type = 'A' - _base3.params['ttl'] = 2 - _base3.params['arecords'] = ARecord('1.1.1.3') - azure_records.append(_base3) - - _base4 = _AzureRecord('TestAzure', octo_records[4]) - _base4.zone_name = 'unit.tests' - _base4.relative_record_set_name = 'cname' - _base4.record_type = 'CNAME' - _base4.params['ttl'] = 3 - _base4.params['cname_record'] = CnameRecord('a.unit.tests.') - azure_records.append(_base4) - - _base5 = _AzureRecord('TestAzure', octo_records[5]) - _base5.zone_name = 'unit.tests' - _base5.relative_record_set_name = '@' - _base5.record_type = 'MX' - _base5.params['ttl'] = 3 - _base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'), - MxRecord(20, 'mx2.unit.tests.')] - azure_records.append(_base5) - - _base6 = _AzureRecord('TestAzure', octo_records[6]) - _base6.zone_name = 'unit.tests' - _base6.relative_record_set_name = '@' - _base6.record_type = 'NS' - _base6.params['ttl'] = 4 - _base6.params['ns_records'] = [NsRecord('ns1.unit.tests.'), - NsRecord('ns2.unit.tests.')] - azure_records.append(_base6) - - _base7 = _AzureRecord('TestAzure', octo_records[7]) - _base7.zone_name = 'unit.tests' - _base7.relative_record_set_name = '@' - _base7.record_type = 'NS' - _base7.params['ttl'] = 5 - _base7.params['ns_records'] = [NsRecord('ns1.unit.tests.')] - azure_records.append(_base7) - - _base8 = _AzureRecord('TestAzure', octo_records[8]) - _base8.zone_name = 'unit.tests' - _base8.relative_record_set_name = '_srv._tcp' - _base8.record_type = 'SRV' - _base8.params['ttl'] = 6 - _base8.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'), - SrvRecord(12, 30, 30, 'foo-2.unit.tests.')] - azure_records.append(_base8) - def test_azure_record(self): - assert(len(self.azure_records) == len(self.octo_records)) - for i in range(len(self.azure_records)): - octo = _AzureRecord('TestAzure', self.octo_records[i]) - assert(self.azure_records[i]._equals(octo)) + assert(len(azure_records) == len(octo_records)) + for i in range(len(azure_records)): + octo = _AzureRecord('TestAzure', octo_records[i]) + assert(azure_records[i]._equals(octo)) + string = str(azure_records[i]) + assert(('Ttl: ' in string)) + + +class TestValidatePeriod(TestCase): + def test_validate_per(self): + for expected, test in [['a.', 'a'], + ['a.', 'a.'], + ['foo.bar.', 'foo.bar.'], + ['foo.bar.', 'foo.bar']]: + self.assertEquals(expected, _validate_per(test)) class TestAzureDnsProvider(TestCase): - def test_populate(self): - pass # placeholder + def _provider(self): + return self._get_provider('mock_spc', 'mock_dns_client') + + @patch('octodns.provider.azuredns.DnsManagementClient') + @patch('octodns.provider.azuredns.ServicePrincipalCredentials') + def _get_provider(self, mock_spc, mock_dns_client): + '''Returns a mock AzureProvider object to use in testing. + + :param mock_spc: placeholder + :type mock_spc: str + :param mock_dns_client: placeholder + :type mock_dns_client: str + + :type return: AzureProvider + ''' + return AzureProvider('mock_id', 'mock_client', 'mock_key', + 'mock_directory', 'mock_sub', 'mock_rg') + + def test_populate_records(self): + provider = self._get_provider() + + rs = [] + rs.append(RecordSet(name='a1', ttl=0, type='A', + arecords=[ARecord('1.1.1.1')])) + rs.append(RecordSet(name='a2', ttl=1, type='A', + arecords=[ARecord('1.1.1.1'), + ARecord('2.2.2.2')])) + rs.append(RecordSet(name='aaaa1', ttl=2, type='AAAA', + aaaa_records=[AaaaRecord('1:1ec:1::1')])) + rs.append(RecordSet(name='aaaa2', ttl=3, type='AAAA', + aaaa_records=[AaaaRecord('1:1ec:1::1'), + AaaaRecord('1:1ec:1::2')])) + rs.append(RecordSet(name='cname1', ttl=4, type='CNAME', + cname_record=CnameRecord('cname.unit.test.'))) + rs.append(RecordSet(name='cname2', ttl=5, type='CNAME', + cname_record=None)) + rs.append(RecordSet(name='mx1', ttl=6, type='MX', + mx_records=[MxRecord(10, 'mx1.unit.test.')])) + rs.append(RecordSet(name='mx2', ttl=7, type='MX', + mx_records=[MxRecord(10, 'mx1.unit.test.'), + MxRecord(11, 'mx2.unit.test.')])) + rs.append(RecordSet(name='ns1', ttl=8, type='NS', + ns_records=[NsRecord('ns1.unit.test.')])) + rs.append(RecordSet(name='ns2', ttl=9, type='NS', + ns_records=[NsRecord('ns1.unit.test.'), + NsRecord('ns2.unit.test.')])) + rs.append(RecordSet(name='ptr1', ttl=10, type='PTR', + ptr_records=[PtrRecord('ptr1.unit.test.')])) + rs.append(RecordSet(name='ptr2', ttl=11, type='PTR', + ptr_records=[PtrRecord('ptr1.unit.test.'), + PtrRecord('ptr2.unit.test.')])) + rs.append(RecordSet(name='_srv1._tcp', ttl=12, type='SRV', + srv_records=[SrvRecord(1, 2, 3, '1unit.tests.')])) + rs.append(RecordSet(name='_srv2._tcp', ttl=13, type='SRV', + srv_records=[SrvRecord(1, 2, 3, '1unit.tests.'), + SrvRecord(4, 5, 6, '2unit.tests.')])) + rs.append(RecordSet(name='txt1', ttl=14, type='TXT', + txt_records=[TxtRecord('sample text1')])) + rs.append(RecordSet(name='txt2', ttl=15, type='TXT', + txt_records=[TxtRecord('sample text1'), + TxtRecord('sample text2')])) + rs.append(RecordSet(name='', ttl=16, type='SOA', + soa_record=[SoaRecord()])) + + record_list = provider._dns_client.record_sets.list_by_dns_zone + record_list.return_value = rs + + provider.populate(zone) + + self.assertEquals(len(zone.records), 16) + + def test_populate_zone(self): + provider = self._get_provider() + + zone_list = provider._dns_client.zones.list_by_resource_group + zone_list.return_value = [AzureZone(location='global'), + AzureZone(location='global')] + + provider._populate_zones() + + self.assertEquals(len(provider._azure_zones), 1) + + def test_bad_zone_response(self): + provider = self._get_provider() + + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), 'Azure Error') + trip = False + try: + provider._check_zone('unit.test', create=False) + except CloudError: + trip = True + self.assertEquals(trip, True) + + def test_apply(self): + provider = self._get_provider() + + changes = [] + deletes = [] + for i in octo_records: + changes.append(Create(i)) + deletes.append(Delete(i)) + + self.assertEquals(11, provider.apply(Plan(None, zone, changes))) + self.assertEquals(11, provider.apply(Plan(zone, zone, deletes))) + + def test_create_zone(self): + provider = self._get_provider() + + changes = [] + for i in octo_records: + changes.append(Create(i)) + desired = Zone('unit2.test.', []) + + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), 'Azure Error') + + self.assertEquals(11, provider.apply(Plan(None, desired, changes))) From 824cf4e98c3cd9e334fe9def9402dfefdb4abc16 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 30 Jun 2017 17:12:18 -0700 Subject: [PATCH 12/24] Changed code as per PR review. Only major change is refactoring _check_zones. Many more comments --- .gitignore | 2 - doit.txt | 4 - octodns/provider/azuredns.py | 146 ++++++++++++++++-------- tests/test_octodns_provider_azuredns.py | 43 ++++++- 4 files changed, 138 insertions(+), 57 deletions(-) delete mode 100755 doit.txt diff --git a/.gitignore b/.gitignore index 5a3f3f9..c45a684 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,3 @@ output/ tmp/ build/ config/ -./rb.txt -./doit.txt diff --git a/doit.txt b/doit.txt deleted file mode 100755 index 1ff2da6..0000000 --- a/doit.txt +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -#script to rebuild octodns quickly - -octodns-sync --config-file=./config/production.yaml --doit diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index cf0e2d5..91e298c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -9,6 +9,7 @@ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone +from msrestazure.azure_exceptions import CloudError from functools import reduce import sys @@ -18,7 +19,8 @@ from .base import BaseProvider class _AzureRecord(object): - ''' Wrapper for OctoDNS record. + '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. + azuredns.py: class: octodns.provider.azuredns._AzureRecord An _AzureRecord is easily accessible to Azure DNS Management library @@ -27,7 +29,19 @@ class _AzureRecord(object): ''' def __init__(self, resource_group, record, delete=False): - ''' + '''Contructor for _AzureRecord. + + Notes on Azure records: An Azure record set has the form + RecordSet(name=<...>, type=<...>, arecords=[...], aaaa_records, ..) + When constructing an azure record as done in self._apply_Create, + the argument parameters for an A record would be + parameters={'ttl': , 'arecords': [ARecord(),]}. + As another example for CNAME record: + parameters={'ttl': , 'cname_record': CnameRecord()}. + + Below, key_name and class_name are the dictionary key and Azure + Record class respectively. + :param resource_group: The name of resource group in Azure :type resource_group: str :param record: An OctoDNS record @@ -39,15 +53,18 @@ class _AzureRecord(object): :type return: _AzureRecord ''' self.resource_group = resource_group - self.zone_name = record.zone.name[0:len(record.zone.name) - 1] + self.zone_name = record.zone.name[:len(record.zone.name) - 1] self.relative_record_set_name = record.name or '@' self.record_type = record._type if delete: return + # Refer to function docstring for key_name and class_name. format_u_s = '' if record._type == 'A' else '_' key_name = '{}{}records'.format(self.record_type, format_u_s).lower() + if record._type == 'CNAME': + key_name = key_name[:len(key_name) - 1] class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) @@ -56,8 +73,10 @@ class _AzureRecord(object): self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): - return {key_name: [azure_class(v) for v in data['values']]} \ - if 'values' in data else {key_name: [azure_class(data['value'])]} + if 'values' in data: + return {key_name: [azure_class(v) for v in data['values']]} + else: # Else there is a singular data point keyed by 'value'. + return {key_name: [azure_class(data['value'])]} _params_for_A = _params _params_for_AAAA = _params @@ -73,7 +92,7 @@ class _AzureRecord(object): vals['weight'], vals['port'], vals['target'])) - else: # single value at key 'value' + else: # Else there is a singular data point keyed by 'value'. params.append(azure_class(data['value']['priority'], data['value']['weight'], data['value']['port'], @@ -86,7 +105,7 @@ class _AzureRecord(object): for vals in data['values']: params.append(azure_class(vals['preference'], vals['exchange'])) - else: # single value at key 'value' + else: # Else there is a singular data point keyed by 'value'. params.append(azure_class(data['value']['preference'], data['value']['exchange'])) return {key_name: params} @@ -141,10 +160,14 @@ class _AzureRecord(object): return string -def _validate_per(string): +def _check_endswith_dot(string): return string if string.endswith('.') else string + '.' +def _parse_azure_type(string): + return string.split('/')[len(string.split('/')) - 1] + + class AzureProvider(BaseProvider): ''' Azure DNS Provider @@ -155,21 +178,44 @@ class AzureProvider(BaseProvider): # includes using a Service Principal: # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ # resource-group-create-service-principal-portal - # The Azure Active Directory Application ID (aka client ID) req: + # The Azure Active Directory Application ID (aka client ID): client_id: - # Authentication Key Value req: + # Authentication Key Value: (note this should be secret) key: - # Directory ID (referred to tenant ID) req: + # Directory ID (aka tenant ID): directory_id: - # Subscription ID req: + # Subscription ID: sub_id: - # Resource Group name req: + # Resource Group name: resource_group: + # All are required to authenticate. - TODO: change config file to use env vars instead of hard-coded keys + Example config file with variables: + " + --- + providers: + config: + class: octodns.provider.yaml.YamlProvider + directory: ./config (example path to directory of zone files) + azuredns: + class: octodns.provider.azuredns.AzureProvider + client_id: env/AZURE_APPLICATION_ID + key: env/AZURE_AUTHENICATION_KEY + directory_id: env/AZURE_DIRECTORY_ID + sub_id: env/AZURE_SUBSCRIPTION_ID + resource_group: 'TestResource1' - personal notes: testing: test authentication vars located in - /home/t-hehwan/vars.txt + zones: + example.com.: + sources: + - config + targets: + - azuredns + " + The first four variables above can be hidden in environment variables + and octoDNS will automatically search for them in the shell. It is + possible to also hard-code into the config file. resource_group can + also be an environment variable but might likely change. ''' SUPPORTS_GEO = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) @@ -214,19 +260,24 @@ class AzureProvider(BaseProvider): self._dns_client.zones.get(self._resource_group, name) self._azure_zones.add(name) return name - except: - if create: - self.log.debug('_check_zone:no matching zone; creating %s', - name) - create_zone = self._dns_client.zones.create_or_update - create_zone(self._resource_group, name, Zone('global')) - return name - else: - raise + except CloudError as err: + msg = 'The Resource \'Microsoft.Network/dnszones/{}\''.format(name) + msg += ' under resource group \'{}\''.format(self._resource_group) + msg += ' was not found.' + if msg == err.message: + # Then the only error is that the zone doesn't currently exist + if create: + self.log.debug('_check_zone:no matching zone; creating %s', + name) + create_zone = self._dns_client.zones.create_or_update + create_zone(self._resource_group, name, Zone('global')) + return name + else: + return + raise def populate(self, zone, target=False, lenient=False): - ''' - Required function of manager.py. + '''Required function of manager.py to collect records from zone. Special notes for Azure. Azure zone names omit final '.' @@ -249,26 +300,28 @@ class AzureProvider(BaseProvider): :type return: void ''' - zone_name = zone.name[0:len(zone.name) - 1] - self.log.debug('populate: name=%s', zone_name) + self.log.debug('populate: name=%s', zone.name) before = len(zone.records) + zone_name = zone.name[:len(zone.name) - 1] self._populate_zones() self._check_zone(zone_name) _records = set() records = self._dns_client.record_sets.list_by_dns_zone - for azrecord in records(self._resource_group, zone_name): - if azrecord.type in self.SUPPORTS: - _records.add(azrecord) - for azrecord in _records: - record_name = azrecord.name if azrecord.name != '@' else '' - data = getattr(self, '_data_for_{}'.format(azrecord.type)) - data = data(azrecord) - data['type'] = azrecord.type - data['ttl'] = azrecord.ttl - record = Record.new(zone, record_name, data, source=self) - zone.add_record(record) + if self._check_zone(zone_name): + for azrecord in records(self._resource_group, zone_name): + if _parse_azure_type(azrecord.type) in self.SUPPORTS: + _records.add(azrecord) + for azrecord in _records: + record_name = azrecord.name if azrecord.name != '@' else '' + typ = _parse_azure_type(azrecord.type) + data = getattr(self, '_data_for_{}'.format(typ)) + data = data(azrecord) + data['type'] = typ + data['ttl'] = azrecord.ttl + record = Record.new(zone, record_name, data, source=self) + zone.add_record(record) self.log.info('populate: found %s records', len(zone.records) - before) @@ -293,13 +346,14 @@ class AzureProvider(BaseProvider): records. Refer to population comment. ''' try: - return {'value': _validate_per(azrecord.cname_record.cname)} + return {'value': _check_endswith_dot(azrecord.cname_record.cname)} except: return {'value': '.'} def _data_for_PTR(self, azrecord): try: - return {'value': _validate_per(azrecord.ptr_records[0].ptdrname)} + ptdrname = azrecord.ptr_records[0].ptdrname + return {'value': _check_endswith_dot(ptdrname)} except: return {'value': '.'} @@ -315,7 +369,7 @@ class AzureProvider(BaseProvider): def _data_for_NS(self, azrecord): vals = [ar.nsdname for ar in azrecord.ns_records] - return {'values': [_validate_per(val) for val in vals]} + return {'values': [_check_endswith_dot(val) for val in vals]} def _apply_Create(self, change): '''A record from change must be created. @@ -334,7 +388,7 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - self.log.debug('* Success Create/Update: {}'.format(ar)) + print('* Success Create/Update: {}'.format(ar), file=sys.stderr) _apply_Update = _apply_Create @@ -345,7 +399,7 @@ class AzureProvider(BaseProvider): delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - self.log.debug('* Success Delete: {}'.format(ar)) + print('* Success Delete: {}'.format(ar), file=sys.stderr) def _apply(self, plan): ''' @@ -361,7 +415,7 @@ class AzureProvider(BaseProvider): self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - azure_zone_name = desired.name[0:len(desired.name) - 1] + azure_zone_name = desired.name[:len(desired.name) - 1] self._check_zone(azure_zone_name, create=True) for change in changes: diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index edb2db2..abce5aa 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from octodns.record import Create, Delete, Record from octodns.provider.azuredns import _AzureRecord, AzureProvider, \ - _validate_per + _check_endswith_dot, _parse_azure_type from octodns.zone import Zone from octodns.provider.base import Plan @@ -195,13 +195,22 @@ class Test_AzureRecord(TestCase): assert(('Ttl: ' in string)) -class TestValidatePeriod(TestCase): - def test_validate_per(self): +class Test_ParseAzureType(TestCase): + def test_parse_azure_type(self): + for expected, test in [['A', 'Microsoft.Network/dnszones/A'], + ['AAAA', 'Microsoft.Network/dnszones/AAAA'], + ['NS', 'Microsoft.Network/dnszones/NS'], + ['MX', 'Microsoft.Network/dnszones/MX']]: + self.assertEquals(expected, _parse_azure_type(test)) + + +class Test_CheckEndswithDot(TestCase): + def test_check_endswith_dot(self): for expected, test in [['a.', 'a'], ['a.', 'a.'], ['foo.bar.', 'foo.bar.'], ['foo.bar.', 'foo.bar']]: - self.assertEquals(expected, _validate_per(test)) + self.assertEquals(expected, _check_endswith_dot(test)) class TestAzureDnsProvider(TestCase): @@ -319,7 +328,31 @@ class TestAzureDnsProvider(TestCase): changes.append(Create(i)) desired = Zone('unit2.test.', []) + err_msg = 'The Resource \'Microsoft.Network/dnszones/unit2.test\' ' + err_msg += 'under resource group \'mock_rg\' was not found.' _get = provider._dns_client.zones.get - _get.side_effect = CloudError(Mock(status=404), 'Azure Error') + _get.side_effect = CloudError(Mock(status=404), err_msg) self.assertEquals(11, provider.apply(Plan(None, desired, changes))) + + def test_check_zone_no_create(self): + provider = self._get_provider() + + rs = [] + rs.append(RecordSet(name='a1', ttl=0, type='A', + arecords=[ARecord('1.1.1.1')])) + rs.append(RecordSet(name='a2', ttl=1, type='A', + arecords=[ARecord('1.1.1.1'), + ARecord('2.2.2.2')])) + + record_list = provider._dns_client.record_sets.list_by_dns_zone + record_list.return_value = rs + + err_msg = 'The Resource \'Microsoft.Network/dnszones/unit3.test\' ' + err_msg += 'under resource group \'mock_rg\' was not found.' + _get = provider._dns_client.zones.get + _get.side_effect = CloudError(Mock(status=404), err_msg) + + provider.populate(Zone('unit3.test.', [])) + + self.assertEquals(len(zone.records), 0) From ec4261e7da97bf8711c830965f18f7da3e725439 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Fri, 30 Jun 2017 17:29:16 -0700 Subject: [PATCH 13/24] Fixed typed in _data_for_PTR and amended test case to check for it --- octodns/provider/azuredns.py | 4 ++-- tests/test_octodns_provider_azuredns.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 91e298c..4fe5184 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -352,8 +352,8 @@ class AzureProvider(BaseProvider): def _data_for_PTR(self, azrecord): try: - ptdrname = azrecord.ptr_records[0].ptdrname - return {'value': _check_endswith_dot(ptdrname)} + ptrdname = azrecord.ptr_records[0].ptrdname + return {'value': _check_endswith_dot(ptrdname)} except: return {'value': '.'} diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index abce5aa..5ffa055 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -263,8 +263,7 @@ class TestAzureDnsProvider(TestCase): rs.append(RecordSet(name='ptr1', ttl=10, type='PTR', ptr_records=[PtrRecord('ptr1.unit.test.')])) rs.append(RecordSet(name='ptr2', ttl=11, type='PTR', - ptr_records=[PtrRecord('ptr1.unit.test.'), - PtrRecord('ptr2.unit.test.')])) + ptr_records=[PtrRecord(None)])) rs.append(RecordSet(name='_srv1._tcp', ttl=12, type='SRV', srv_records=[SrvRecord(1, 2, 3, '1unit.tests.')])) rs.append(RecordSet(name='_srv2._tcp', ttl=13, type='SRV', From bdceac42beb485ed30417a1ae85db1f6e45673b2 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Sun, 2 Jul 2017 18:40:58 -0700 Subject: [PATCH 14/24] Fix stacktraces on MainThreadExecutor --- octodns/manager.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/octodns/manager.py b/octodns/manager.py index e6fe253..8439eb6 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals from StringIO import StringIO -from concurrent.futures import Future, ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor from importlib import import_module from os import environ import logging @@ -38,6 +38,17 @@ class _AggregateTarget(object): return True +class MakeThreadFuture(object): + + def __init__(self, func, args, kwargs): + self.func = func + self.args = args + self.kwargs = kwargs + + def result(self): + return self.func(*self.args, **self.kwargs) + + class MainThreadExecutor(object): ''' Dummy executor that runs things on the main thread during the involcation @@ -48,13 +59,7 @@ class MainThreadExecutor(object): ''' def submit(self, func, *args, **kwargs): - future = Future() - try: - future.set_result(func(*args, **kwargs)) - except Exception as e: - # TODO: get right stacktrace here - future.set_exception(e) - return future + return MakeThreadFuture(func, args, kwargs) class Manager(object): From 9637218c8b544c397bcd5d433de47cafbfad973d Mon Sep 17 00:00:00 2001 From: Petter Hassberg Date: Mon, 3 Jul 2017 16:10:48 +0200 Subject: [PATCH 15/24] Add lenient to abstract BaseSource signature --- octodns/source/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/source/base.py b/octodns/source/base.py index 2e2c5c2..4ace09f 100644 --- a/octodns/source/base.py +++ b/octodns/source/base.py @@ -20,7 +20,7 @@ class BaseSource(object): raise NotImplementedError('Abstract base class, SUPPORTS ' 'property missing') - def populate(self, zone, target=False): + def populate(self, zone, target=False, lenient=False): ''' Loads all zones the provider knows about From cd9d7254f06b1f5d24524bae0e236367642ff899 Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 3 Jul 2017 11:03:23 -0700 Subject: [PATCH 16/24] Fixed stray prints and assert errors. Added versions of required azure libraries --- octodns/provider/azuredns.py | 30 +++++++++++++++++++++--------- requirements.txt | 3 +++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4fe5184..0abddb0 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -7,17 +7,30 @@ from __future__ import absolute_import, division, print_function, \ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient -from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ - SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone from msrestazure.azure_exceptions import CloudError -from functools import reduce -import sys +# Imports are used: 'self.params(record.data, key_name, eval(class_name))' +# To pass pyflakes import statement tests. +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord +from azure.mgmt.dns.models import Zone + import logging +from functools import reduce from ..record import Record from .base import BaseProvider +ARecord +AaaaRecord +CnameRecord +MxRecord +SrvRecord +NsRecord +PtrRecord +TxtRecord + + class _AzureRecord(object): '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. @@ -65,8 +78,7 @@ class _AzureRecord(object): key_name = '{}{}records'.format(self.record_type, format_u_s).lower() if record._type == 'CNAME': key_name = key_name[:len(key_name) - 1] - class_name = '{}'.format(self.record_type).capitalize() + \ - 'Record'.format(self.record_type) + class_name = '{}'.format(self.record_type).capitalize() + 'Record' self.params = getattr(self, '_params_for_{}'.format(record._type)) self.params = self.params(record.data, key_name, eval(class_name)) @@ -111,7 +123,7 @@ class _AzureRecord(object): return {key_name: params} def _params_for_CNAME(self, data, key_name, azure_class): - return {'cname_record': CnameRecord(data['value'])} + return {'cname_record': azure_class(data['value'])} def _equals(self, b): '''Checks whether two records are equal by comparing all fields. @@ -388,7 +400,7 @@ class AzureProvider(BaseProvider): record_type=ar.record_type, parameters=ar.params) - print('* Success Create/Update: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Create/Update: {}'.format(ar)) _apply_Update = _apply_Create @@ -399,7 +411,7 @@ class AzureProvider(BaseProvider): delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) - print('* Success Delete: {}'.format(ar), file=sys.stderr) + self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): ''' diff --git a/requirements.txt b/requirements.txt index b10ca4c..62c485a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,6 @@ python-dateutil==2.6.0 requests==2.13.0 s3transfer==0.1.10 six==1.10.0 +azure-mgmt-dns==1.0.1 +azure-common==1.1.6 +msrestazure==0.4.10 \ No newline at end of file From 348a6ca783af214632831961abd0a5aec8a2225a Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Mon, 3 Jul 2017 16:03:20 -0700 Subject: [PATCH 17/24] Changed to map types to Azure Records isntead of implicitly using eval --- octodns/provider/azuredns.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 0abddb0..4ed783c 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -9,11 +9,8 @@ from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient from msrestazure.azure_exceptions import CloudError -# Imports are used: 'self.params(record.data, key_name, eval(class_name))' -# To pass pyflakes import statement tests. from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ - SrvRecord, NsRecord, PtrRecord, TxtRecord -from azure.mgmt.dns.models import Zone + SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone import logging from functools import reduce @@ -21,16 +18,6 @@ from ..record import Record from .base import BaseProvider -ARecord -AaaaRecord -CnameRecord -MxRecord -SrvRecord -NsRecord -PtrRecord -TxtRecord - - class _AzureRecord(object): '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls. @@ -40,6 +27,16 @@ class _AzureRecord(object): functions and is used to wrap all relevant data to create a record in Azure. ''' + TYPE_MAP = { + 'A': ARecord, + 'AAAA': AaaaRecord, + 'CNAME': CnameRecord, + 'MX': MxRecord, + 'SRV': SrvRecord, + 'NS': NsRecord, + 'PTR': PtrRecord, + 'TXT': TxtRecord + } def __init__(self, resource_group, record, delete=False): '''Contructor for _AzureRecord. @@ -78,10 +75,10 @@ class _AzureRecord(object): key_name = '{}{}records'.format(self.record_type, format_u_s).lower() if record._type == 'CNAME': key_name = key_name[:len(key_name) - 1] - class_name = '{}'.format(self.record_type).capitalize() + 'Record' + azure_class = self.TYPE_MAP[self.record_type] self.params = getattr(self, '_params_for_{}'.format(record._type)) - self.params = self.params(record.data, key_name, eval(class_name)) + self.params = self.params(record.data, key_name, azure_class) self.params['ttl'] = record.ttl def _params(self, data, key_name, azure_class): From 99578f328c18fc72f874da090952d7cd5e040e2f Mon Sep 17 00:00:00 2001 From: Heesu Hwang Date: Wed, 5 Jul 2017 09:45:38 -0700 Subject: [PATCH 18/24] add azure to README. order reqs, change comments slightly, alphabetize functions --- README.md | 1 + octodns/provider/azuredns.py | 76 +++++++++++++------------ requirements.txt | 8 +-- tests/test_octodns_provider_azuredns.py | 1 - 4 files changed, 45 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 0e63e51..05cf979 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,7 @@ The above command pulled the existing data out of Route53 and placed the results | Provider | Record Support | GeoDNS Support | Notes | |--|--|--|--| +| [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CNAME, MX, NS, SPF, TXT | No | | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 4ed783c..4433b1e 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -56,9 +56,8 @@ class _AzureRecord(object): :type resource_group: str :param record: An OctoDNS record :type record: ..record.Record - :param values: Parameters for a record. eg IP address, port, domain - name, etc. Values usually read from record.data - :type values: {'values': [...]} or {'value': [...]} + :param delete: If true, omit data parsing; not needed to delete + :type delete: bool :type return: _AzureRecord ''' @@ -93,6 +92,20 @@ class _AzureRecord(object): _params_for_PTR = _params _params_for_TXT = _params + def _params_for_CNAME(self, data, key_name, azure_class): + return {key_name: azure_class(data['value'])} + + def _params_for_MX(self, data, key_name, azure_class): + params = [] + if 'values' in data: + for vals in data['values']: + params.append(azure_class(vals['preference'], + vals['exchange'])) + else: # Else there is a singular data point keyed by 'value'. + params.append(azure_class(data['value']['preference'], + data['value']['exchange'])) + return {key_name: params} + def _params_for_SRV(self, data, key_name, azure_class): params = [] if 'values' in data: @@ -108,20 +121,6 @@ class _AzureRecord(object): data['value']['target'])) return {key_name: params} - def _params_for_MX(self, data, key_name, azure_class): - params = [] - if 'values' in data: - for vals in data['values']: - params.append(azure_class(vals['preference'], - vals['exchange'])) - else: # Else there is a singular data point keyed by 'value'. - params.append(azure_class(data['value']['preference'], - data['value']['exchange'])) - return {key_name: params} - - def _params_for_CNAME(self, data, key_name, azure_class): - return {'cname_record': azure_class(data['value'])} - def _equals(self, b): '''Checks whether two records are equal by comparing all fields. :param b: Another _AzureRecord object @@ -174,6 +173,13 @@ def _check_endswith_dot(string): def _parse_azure_type(string): + '''Converts string representing an Azure RecordSet type to usual type. + + :param string: the Azure type. eg: + :type string: str + + :type return: str + ''' return string.split('/')[len(string.split('/')) - 1] @@ -223,8 +229,7 @@ class AzureProvider(BaseProvider): " The first four variables above can be hidden in environment variables and octoDNS will automatically search for them in the shell. It is - possible to also hard-code into the config file. resource_group can - also be an environment variable but might likely change. + possible to also hard-code into the config file: eg, resource_group. ''' SUPPORTS_GEO = False SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) @@ -250,8 +255,8 @@ class AzureProvider(BaseProvider): self._azure_zones.add(zone.name) def _check_zone(self, name, create=False): - ''' - Checks whether a zone specified in a source exist in Azure server. + '''Checks whether a zone specified in a source exist in Azure server. + Note that Azure zones omit end '.' eg: contoso.com vs contoso.com. Returns the name if it exists. @@ -302,7 +307,7 @@ class AzureProvider(BaseProvider): :param zone: A dns zone :type zone: octodns.zone.Zone :param target: Checks if Azure is source or target of config. - Currently only supports as a target. Does not use. + Currently only supports as a target. Unused. :type target: bool :param lenient: Unused. Check octodns.manager for usage. :type lenient: bool @@ -340,10 +345,6 @@ class AzureProvider(BaseProvider): def _data_for_AAAA(self, azrecord): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} - def _data_for_TXT(self, azrecord): - return {'values': [reduce((lambda a, b: a + b), ar.value) - for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): '''Parsing data from Azure DNS Client record call :param azrecord: a return of a call to list azure records @@ -359,6 +360,15 @@ class AzureProvider(BaseProvider): except: return {'value': '.'} + def _data_for_MX(self, azrecord): + return {'values': [{'preference': ar.preference, + 'exchange': ar.exchange} + for ar in azrecord.mx_records]} + + def _data_for_NS(self, azrecord): + vals = [ar.nsdname for ar in azrecord.ns_records] + return {'values': [_check_endswith_dot(val) for val in vals]} + def _data_for_PTR(self, azrecord): try: ptrdname = azrecord.ptr_records[0].ptrdname @@ -366,19 +376,14 @@ class AzureProvider(BaseProvider): except: return {'value': '.'} - def _data_for_MX(self, azrecord): - return {'values': [{'preference': ar.preference, - 'exchange': ar.exchange} - for ar in azrecord.mx_records]} - def _data_for_SRV(self, azrecord): return {'values': [{'priority': ar.priority, 'weight': ar.weight, 'port': ar.port, 'target': ar.target} for ar in azrecord.srv_records]} - def _data_for_NS(self, azrecord): - vals = [ar.nsdname for ar in azrecord.ns_records] - return {'values': [_check_endswith_dot(val) for val in vals]} + def _data_for_TXT(self, azrecord): + return {'values': [reduce((lambda a, b: a + b), ar.value) + for ar in azrecord.txt_records]} def _apply_Create(self, change): '''A record from change must be created. @@ -411,8 +416,7 @@ class AzureProvider(BaseProvider): self.log.debug('* Success Delete: {}'.format(ar)) def _apply(self, plan): - ''' - Required function of manager.py + '''Required function of manager.py to actually apply a record change. :param plan: Contains the zones and changes to be made :type plan: octodns.provider.base.Plan diff --git a/requirements.txt b/requirements.txt index 9eb8284..5d8089a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ # These are known good versions. You're free to use others and things will # likely work, but no promises are made, especilly if you go older. PyYaml==3.12 +azure-mgmt-dns==1.0.1 +azure-common==1.1.6 boto3==1.4.4 botocore==1.5.4 dnspython==1.15.0 @@ -10,12 +12,10 @@ futures==3.0.5 incf.countryutils==1.0 ipaddress==1.0.18 jmespath==0.9.0 +msrestazure==0.4.10 natsort==5.0.3 nsone==0.9.14 python-dateutil==2.6.0 requests==2.13.0 s3transfer==0.1.10 -six==1.10.0 -azure-mgmt-dns==1.0.1 -azure-common==1.1.6 -msrestazure==0.4.10 \ No newline at end of file +six==1.10.0 \ No newline at end of file diff --git a/tests/test_octodns_provider_azuredns.py b/tests/test_octodns_provider_azuredns.py index 5ffa055..59cf551 100644 --- a/tests/test_octodns_provider_azuredns.py +++ b/tests/test_octodns_provider_azuredns.py @@ -16,7 +16,6 @@ from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ Zone as AzureZone from msrestazure.azure_exceptions import CloudError - from unittest import TestCase from mock import Mock, patch From 8d7b3fb101959898a576e735dea606e616d26614 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 5 Jul 2017 14:09:05 -0700 Subject: [PATCH 19/24] Remove ; escapes before sending to ns1 and when pulling from --- octodns/provider/ns1.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 65db64c..e7b8ffb 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -42,8 +42,16 @@ class Ns1Provider(BaseProvider): } _data_for_AAAA = _data_for_A - _data_for_SPF = _data_for_A - _data_for_TXT = _data_for_A + + def _data_for_SPF(self, _type, record): + values = [v.replace(';', '\;') for v in record['short_answers']] + return { + 'ttl': record['ttl'], + 'type': _type, + 'values': values + } + + _data_for_TXT = _data_for_SPF def _data_for_CNAME(self, _type, record): return { @@ -141,8 +149,15 @@ class Ns1Provider(BaseProvider): _params_for_AAAA = _params_for_A _params_for_NS = _params_for_A - _params_for_SPF = _params_for_A - _params_for_TXT = _params_for_A + + def _params_for_SPF(self, record): + # NS1 seems to be the only provider that doesn't want things escaped in + # values so we have to strip them here and add them when going the + # other way + values = [v.replace('\\', '') for v in record.values] + return {'answers': values, 'ttl': record.ttl} + + _params_for_TXT = _params_for_SPF def _params_for_CNAME(self, record): return {'answers': [record.value], 'ttl': record.ttl} From 818c1e9cc6ba6d155d43a4f3152615d48dabe091 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 5 Jul 2017 14:28:01 -0700 Subject: [PATCH 20/24] Unit tests for ns1 escape handling and fix --- octodns/provider/ns1.py | 2 +- tests/test_octodns_provider_ns1.py | 34 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index e7b8ffb..7757812 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -154,7 +154,7 @@ class Ns1Provider(BaseProvider): # NS1 seems to be the only provider that doesn't want things escaped in # values so we have to strip them here and add them when going the # other way - values = [v.replace('\\', '') for v in record.values] + values = [v.replace('\;', ';') for v in record.values] return {'answers': values, 'ttl': record.ttl} _params_for_TXT = _params_for_SPF diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index 0398459..afcdd41 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -277,3 +277,37 @@ class TestNs1Provider(TestCase): call.update(answers=[u'1.2.3.4'], ttl=32), call.delete() ]) + + def test_escaping(self): + provider = Ns1Provider('test', 'api-key') + + record = { + 'ttl': 31, + 'short_answers': ['foo; bar baz; blip'] + } + self.assertEquals(['foo\; bar baz\; blip'], + provider._data_for_SPF('SPF', record)['values']) + + record = { + 'ttl': 31, + 'short_answers': ['no', 'foo; bar baz; blip', 'yes'] + } + self.assertEquals(['no', 'foo\; bar baz\; blip', 'yes'], + provider._data_for_TXT('TXT', record)['values']) + + zone = Zone('unit.tests.', []) + record = Record.new(zone, 'spf', { + 'ttl': 34, + 'type': 'SPF', + 'value': 'foo\; bar baz\; blip' + }) + self.assertEquals(['foo; bar baz; blip'], + provider._params_for_SPF(record)['answers']) + + record = Record.new(zone, 'txt', { + 'ttl': 35, + 'type': 'TXT', + 'value': 'foo\; bar baz\; blip' + }) + self.assertEquals(['foo; bar baz; blip'], + provider._params_for_TXT(record)['answers']) From 22a05639160bf7c0417d82af78e0ac4df3d43f15 Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Mon, 10 Jul 2017 16:00:50 -0700 Subject: [PATCH 21/24] fix unsafe plan text interpolation --- octodns/provider/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 2fd4349..bcf566a 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -56,8 +56,8 @@ class Plan(object): delete_pcent = self.change_counts['Delete'] / existing_record_count if update_pcent > self.MAX_SAFE_UPDATE_PCENT: - raise UnsafePlan('Too many updates, %s is over %s percent' - '(%s/%s)', + raise UnsafePlan('Too many updates, {} is over {} percent' + '({}/{})'.format, update_pcent, self.MAX_SAFE_UPDATE_PCENT * 100, self.change_counts['Update'], From 2b58e065e8569e336aaec2cd3f759f03ed69867a Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 11 Jul 2017 07:09:20 -0700 Subject: [PATCH 22/24] fix format --- octodns/provider/base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index bcf566a..8c97edb 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -57,18 +57,18 @@ class Plan(object): if update_pcent > self.MAX_SAFE_UPDATE_PCENT: raise UnsafePlan('Too many updates, {} is over {} percent' - '({}/{})'.format, + '({}/{})'.format( update_pcent, self.MAX_SAFE_UPDATE_PCENT * 100, self.change_counts['Update'], - existing_record_count) + existing_record_count)) if delete_pcent > self.MAX_SAFE_DELETE_PCENT: - raise UnsafePlan('Too many deletes, %s is over %s percent' - '(%s/%s)', + raise UnsafePlan('Too many deletes, {} is over {} percent' + '({}/{})'.format( delete_pcent, self.MAX_SAFE_DELETE_PCENT * 100, self.change_counts['Delete'], - existing_record_count) + existing_record_count)) def __repr__(self): return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \ From 11a246da81928640387a8af36f0b3da8a93cfbe2 Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 11 Jul 2017 07:20:18 -0700 Subject: [PATCH 23/24] whitespace --- octodns/provider/base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/octodns/provider/base.py b/octodns/provider/base.py index 8c97edb..d2561ba 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -58,17 +58,17 @@ class Plan(object): if update_pcent > self.MAX_SAFE_UPDATE_PCENT: raise UnsafePlan('Too many updates, {} is over {} percent' '({}/{})'.format( - update_pcent, - self.MAX_SAFE_UPDATE_PCENT * 100, - self.change_counts['Update'], - existing_record_count)) + update_pcent, + self.MAX_SAFE_UPDATE_PCENT * 100, + self.change_counts['Update'], + existing_record_count)) if delete_pcent > self.MAX_SAFE_DELETE_PCENT: raise UnsafePlan('Too many deletes, {} is over {} percent' '({}/{})'.format( - delete_pcent, - self.MAX_SAFE_DELETE_PCENT * 100, - self.change_counts['Delete'], - existing_record_count)) + delete_pcent, + self.MAX_SAFE_DELETE_PCENT * 100, + self.change_counts['Delete'], + existing_record_count)) def __repr__(self): return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \ From 5b746845ed567169edb523a2149d385743c84c0c Mon Sep 17 00:00:00 2001 From: Joe Williams Date: Tue, 11 Jul 2017 07:36:24 -0700 Subject: [PATCH 24/24] add tests --- tests/test_octodns_provider_base.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index bd134bc..e44adc0 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -214,9 +214,11 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_UPDATE_PCENT) + 1)] - with self.assertRaises(UnsafePlan): + with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes).raise_if_unsafe() + self.assertTrue('Too many updates' in ctx.exception.message) + def test_safe_updates_min_existing_pcent(self): # MAX_SAFE_UPDATE_PCENT is safe when more # than MIN_EXISTING_RECORDS exist @@ -260,9 +262,11 @@ class TestBaseProvider(TestCase): for i in range(int(Plan.MIN_EXISTING_RECORDS * Plan.MAX_SAFE_DELETE_PCENT) + 1)] - with self.assertRaises(UnsafePlan): + with self.assertRaises(UnsafePlan) as ctx: Plan(zone, zone, changes).raise_if_unsafe() + self.assertTrue('Too many deletes' in ctx.exception.message) + def test_safe_deletes_min_existing_pcent(self): # MAX_SAFE_DELETE_PCENT is safe when more # than MIN_EXISTING_RECORDS exist