diff --git a/.gitignore b/.gitignore index eca95c9..5a3f3f9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ nosetests.xml octodns.egg-info/ output/ tmp/ -Makefile build/ -config/ \ No newline at end of file +config/ +./rb.txt +./doit.txt diff --git a/doit.txt b/doit.txt new file mode 100755 index 0000000..1ff2da6 --- /dev/null +++ b/doit.txt @@ -0,0 +1,4 @@ +#!/bin/bash +#script to rebuild octodns quickly + +octodns-sync --config-file=./config/production.yaml --doit diff --git a/octodns/provider/azuredns.py b/octodns/provider/azuredns.py index 66ca8e3..07b39fe 100644 --- a/octodns/provider/azuredns.py +++ b/octodns/provider/azuredns.py @@ -8,54 +8,66 @@ import sys from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.dns import DnsManagementClient -from azure.mgmt.dns.models import * +from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \ + SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone + +from functools import reduce -from collections import defaultdict -# from incf.countryutils.transformations import cca_to_ctca2 TODO: add geo sup. import logging import re from ..record import Record, Update from .base import BaseProvider - -#TODO: changes made to master include adding /build, Makefile to .gitignore and -# making Makefile. -# Only made for A records. will have to adjust for more generic params types class _AzureRecord(object): - def __init__(self, resource_group, record, values=None, ttl=1800): - # print('Here4',file=sys.stderr) + ''' + Wrapper for OctoDNS record. + azuredns.py: + class: octodns.provider.azuredns._AzureRecord + An _AzureRecord is easily accessible to the Azure DNS Management library + functions and is used to wrap all relevant data to create a record in + Azure. + ''' + + def __init__(self, resource_group, record, values=None): + ''' + :param resource_group: The name of resource group in Azure + :type resource_group: str + :param record: An OctoDNS record + :type record: ..record.Record + :param values: Parameters for a record. eg IP address, port, domain + name, etc. Values usually read from record.data + :type values: {'values': [...]} or {'value': [...]} + + :type return: _AzureRecord + ''' self.resource_group = resource_group - self.zone_name = record.zone.name[0:len(record.zone.name)-1] # strips last period + self.zone_name = record.zone.name[0:len(record.zone.name)-1] self.relative_record_set_name = record.name or '@' self.record_type = record._type - type_name = '{}records'.format(self.record_type).lower() + data = values or record.data + format_u_s = '' if record._type == 'A' else '_' + key_name ='{}{}records'.format(self.record_type, format_u_s).lower() class_name = '{}'.format(self.record_type).capitalize() + \ 'Record'.format(self.record_type) - if values == None: - return - # TODO: clean up this bit. - data = values or record.data #This should fail if it gets to record.data? It only returns ttl. TODO - - #depending on mult values or not - #TODO: import explicitly. eval() uses for example ARecord from azure.mgmt.dns.models - self.params = {} - try: - self.params = {'ttl':record.ttl or ttl, \ - type_name:[eval(class_name)(ip) for ip in data['values']] or []} - except KeyError: # means that doesn't have multiple values but single value - self.params = {'ttl':record.ttl or ttl, \ - type_name:[eval(class_name)(data['value'])] or []} + self.params = None + if not self.record_type == 'CNAME': + self.params = self._params(data, key_name, eval(class_name)) + else: + self.params = {'cname_record': CnameRecord(data['value'])} + self.params['ttl'] = record.ttl + def _params(self, data, key_name, azure_class): + return {key_name: [azure_class(v) for v in data['values']]} \ + if 'values' in data else {key_name: [azure_class(data['value'])]} - class AzureProvider(BaseProvider): ''' Azure DNS Provider - - azure.py: - class: octodns.provider.azure.AzureProvider + + azuredns.py: + class: octodns.provider.azuredns.AzureProvider # Current support of authentication of access to Azure services only # includes using a Service Principal: # https://docs.microsoft.com/en-us/azure/azure-resource-manager/ @@ -71,90 +83,118 @@ class AzureProvider(BaseProvider): # Resource Group name req: resource_group: - testing: test authentication vars located in /home/t-hehwan/vars.txt + TODO: change the config file to use env variables instead of hard-coded keys? + + personal notes: testing: test authentication vars located in /home/t-hehwan/vars.txt ''' - SUPPORTS_GEO = False # TODO. Will add support as project progresses. + SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT')) - def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, *args, **kwargs): + def __init__(self, id, client_id, key, directory_id, sub_id, resource_group, + *args, **kwargs): self.log = logging.getLogger('AzureProvider[{}]'.format(id)) self.log.debug('__init__: id=%s, client_id=%s, ' 'key=***, directory_id:%s', id, client_id, directory_id) super(AzureProvider, self).__init__(id, *args, **kwargs) credentials = ServicePrincipalCredentials( - client_id = client_id, secret = key, tenant = directory_id + client_id, secret = key, tenant = directory_id ) self._dns_client = DnsManagementClient(credentials, sub_id) self._resource_group = resource_group - - self._azure_zones = None # will be a dictionary. key: name. val: id. - self._azure_records = {} # will be dict by octodns record, az record - - self._supported_types = ['CNAME', 'A', 'AAAA', 'MX', 'SRV', 'NS', 'PTR'] - # TODO: add TXT - - # TODO: health checks a la route53. - - + self._azure_zones = set() - # TODO: add support for all types. First skeleton: add A. - def supports(self, record): - return record._type in self._supported_types - - @property - def azure_zones(self): - if self._azure_zones is None: - self.log.debug('azure_zones: loading') - zones = {} - for zone in self._dns_client.zones.list_by_resource_group(self._resource_group): - zones[zone.name] = zone.id - self._azure_zones = zones - return self._azure_zones - - # Given a zone name, returns the zone id. If DNE, creates it. - def _get_zone_id(self, name, create=False): - self.log.debug('_get_zone_id: name=%s', name) + def _populate_zones(self): + self.log.debug('azure_zones: loading') + for zone in self._dns_client.zones.list_by_resource_group( + self._resource_group): + self._azure_zones.add(zone.name) + + def _check_zone(self, name, create=False): + ''' + Checks whether a zone specified in a source exist in Azure server. + Note that Azure zones omit end '.' eg: contoso.com vs contoso.com. + Returns the name if it exists. + + :param name: Name of a zone to checks + :type name: str + :param create: If True, creates the zone of that name. + :type create: bool + + :type return: str or None + ''' + self.log.debug('_check_zone: name=%s', name) try: - id = self._dns_client.zones.get(self._resource_group, name) - self.log.debug('_get_zone_id: id=%s', id) - return id + if name in self._azure_zones: + return name + if self._dns_client.zones.get(self._resource_group, name): + self._azure_zones.add(name) + return name except: if create: - self.log.debug('_get_zone_id: no matching zone; creating %s', name) - #TODO: write - return None #placeholder - return None + try: + self.log.debug('_check_zone: no matching zone; creating %s', + name) + if self._dns_client.zones.create_or_update( + self._resource_group, name, Zone('global')): #TODO: figure out what location should be + return name + except: + raise + return None - # Create a dictionary of record objects by zone and octodns record names - # TODO: add geo parsing - def populate(self, zone, target): - zone_name = zone.name[0:len(zone.name)-1]#Azure zone names do not include suffix . + def populate(self, zone, target=False): + ''' + Required function of manager.py. + + Special notes for Azure. Azure zone names omit final '.' + Azure record names for '' are represented by '@' + Azure records created through online interface may have null values + (eg, no IP address for A record). Specific quirks such as these are + responsible for any strange parsing. + + :param zone: A dns zone + :type zone: octodns.zone.Zone + :param target: Checks if Azure is source or target of config. + Currently only supports as a target. Does not use. + :type target: bool + + + TODO: azure interface allows null values. If this attempts to populate with them, will fail. add safety check (simply delete records with null values?) + + :type return: void + ''' + zone_name = zone.name[0:len(zone.name)-1] self.log.debug('populate: name=%s', zone_name) before = len(zone.records) - zone_id = self._get_zone_id(zone_name) - if zone_id: - #records = defaultdict(list) - for type in self._supported_types: - # print('populate. type: {}'.format(type),file=sys.stderr) - for azrecord in self._dns_client.record_sets.list_by_type(self._resource_group, zone_name, type): - # print(azrecord, file=sys.stderr) + + self._populate_zones() + if self._check_zone(zone_name): + for typ in self.SUPPORTS: + for azrecord in self._dns_client.record_sets.list_by_type( + self._resource_group, zone_name, typ): record_name = azrecord.name if azrecord.name != '@' else '' - data = self._type_and_ttl(type, azrecord, - getattr(self, '_data_for_{}'.format(type))(azrecord)) # TODO: azure online interface allows None values. must validate. + data = self._type_and_ttl(typ, azrecord.ttl, + getattr(self, '_data_for_{}'.format(typ))(azrecord)) + record = Record.new(zone, record_name, data, source=self) - # print('HERE0',file=sys.stderr) zone.add_record(record) - self._azure_records[record] = _AzureRecord(self._resource_group, record, data) - # print('HERE1',file=sys.stderr) + self.log.info('populate: found %s records', len(zone.records)-before) - - # might not need - def _get_type(azrecord): - azrecord['type'].split('/')[-1] - - def _type_and_ttl(self, type, azrecord, data): - data['type'] = type - data['ttl'] = azrecord.ttl + + def _type_and_ttl(self, typ, ttl, data): + ''' Adds type and ttl fields to return dictionary. + + :param typ: The type of a record + :type typ: str + :param ttl: The ttl of a record + :type ttl: int + :param data: Dictionary holding values of a record. eg, IP addresses + :type data: {'values': [...]} or {'value': [...]} + + :type return: {...} + ''' + data['type'] = typ + data['ttl'] = ttl return data def _data_for_A(self, azrecord): @@ -164,12 +204,10 @@ class AzureProvider(BaseProvider): return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]} def _data_for_TXT(self, azrecord): - print('azure',file=sys.stderr) - print([ar.value for ar in azrecord.txt_records], file=sys.stderr) - print('',file=sys.stderr) - return {'values': [ar.value for ar in azrecord.txt_records]} + return {'values': \ + [reduce((lambda a,b:a+b), ar.value) for ar in azrecord.txt_records]} - def _data_for_CNAME(self, azrecord): + def _data_for_CNAME(self, azrecord): #TODO: see TODO in population comment. try: val = azrecord.cname_record.cname if not val.endswith('.'): @@ -178,7 +216,7 @@ class AzureProvider(BaseProvider): except: return {'value': '.'} #TODO: this is a bad fix. but octo checks that cnames have trailing '.' while azure allows creating cnames on the online interface with no value. - def _data_for_PTR(self, azrecord): + def _data_for_PTR(self, azrecord): #TODO: see TODO in population comment. try: val = azrecord.ptr_records[0].ptdrname if not val.endswith('.'): @@ -198,21 +236,23 @@ class AzureProvider(BaseProvider): 'target': ar.target} for ar in azrecord.srv_records] } - def _data_for_NS(self, azrecord): + def _data_for_NS(self, azrecord): #TODO: see TODO in population comment. def period_validate(string): return string if string.endswith('.') else string + '.' vals = [ar.nsdname for ar in azrecord.ns_records] return {'values': [period_validate(val) for val in vals]} def _apply_Create(self, change): - new = change.new - - #validate that the zone exists. - #self._get_zone_id(new.name, create=True) - - ar = _AzureRecord(self._resource_group, new, new.data) - + ''' A record from change must be created. + + :param change: a change object + :type change: octodns.record.Change + + :type return: void + ''' + ar = _AzureRecord(self._resource_group, change.new) create = self._dns_client.record_sets.create_or_update + create(resource_group_name=ar.resource_group, zone_name=ar.zone_name, relative_record_set_name=ar.relative_record_set_name, @@ -220,56 +260,33 @@ class AzureProvider(BaseProvider): parameters=ar.params) def _apply_Delete(self, change): - existing = change.existing - ar = _AzureRecord(self._resource_group, existing) + ar = _AzureRecord(self._resource_group, change.existing) delete = self._dns_client.record_sets.delete + delete(self._resource_group, ar.zone_name, ar.relative_record_set_name, ar.record_type) def _apply_Update(self, change): - self._apply_Delete(change) self._apply_Create(change) - # type plan: Plan class from .base def _apply(self, plan): + ''' + Required function of manager.py + + :param plan: Contains the zones and changes to be made + :type plan: octodns.provider.base.Plan + + :type return: void + ''' desired = plan.desired changes = plan.changes self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, len(changes)) - # validate that the zone exists. function creates zone if DNE. - self._get_zone_id(desired.name) + azure_zone_name = desired.name[0:len(desired.name)-1] + self._check_zone(azure_zone_name, create=True) - # Some parsing bits to call _mod_Create or _mod_Delete. - # changes is a list of Delete and Create objects. for change in changes: class_name = change.__class__.__name__ getattr(self, '_apply_{}'.format(class_name))(change) - - - # ********** - # Figuring out what object plan is. - - # self._executor = ThreadPoolExecutor(max_workers) - # futures.append(self._executor.submit(self._populate_and_plan, - # zone_name, sources, targets)) - # plans = [p for f in futures for p in f.results()] - # type of plans[0] == type of one output of _populate_and_plan - - # for target, plan in plans: - # apply(plan) - - - # type(target) == BaseProvider - # type(plan) == Plan() - - # Plan(existing, desired, changes) - # existing.type == desired.type == Zone(desired.name, desired.sub_zones) - # Zone(name, sub_zones) (str and set of strs) - # changes.type = [Delete/Create] - - - # Starts with sync in main() of sync. - # {u'values': ['3.3.3.3', '4.4.4.4'], u'type': 'A', u'ttl': 3600} - # {u'type': u'A', u'value': [u'3.3.3.3', u'4.4.4.4'], u'ttl': 3600L} \ No newline at end of file