Merge remote-tracking branch 'origin/master' into configurable-geo-healthcheck

.git_hooks_pre-commit

@@ -6,6 +6,6 @@ HOOKS=`dirname $0`
 GIT=`dirname $HOOKS`
 ROOT=`dirname $GIT`
 
-source $ROOT/env/bin/activate
+. $ROOT/env/bin/activate
 $ROOT/script/lint
 $ROOT/script/test

.gitignore

@@ -9,3 +9,5 @@ nosetests.xml
 octodns.egg-info/
 output/
 tmp/
+build/
+config/

CHANGELOG.md

@@ -0,0 +1,68 @@
+## v0.8.7 - 2017-09-29 - OVH support
+
+Adds an OVH provider.
+
+## v0.8.6 - 2017-09-06 - CAA record type,
+
+Misc fixes and improvments.
+
+* Azure TXT record fix
+* PowerDNS api support for https
+* Configurable Route53 max retries and max-attempts
+* Improved key ordering error message
+
+## v0.8.5 - 2017-07-21 - Azure, NS1 escaping, & large zones
+
+Relatively small delta this go around. No major themes or anything, just steady
+progress.
+
+* AzureProvider added thanks to work by
+  [Heesu Hwang](https://github.com/h-hwang).
+* Fixed some escaping issues with NS1 TXT and SPF records that were tracked down
+  with the help of [Blake Stoddard](https://github.com/blakestoddard).
+* Some tweaks were made to Zone.records to vastly improve handling of zones with
+  very large numbers of records, no more O(N^2).
+
+## v0.8.4 - 2017-06-28 - It's been too long
+
+Lots of updates based on our internal use, needs, and feedback & suggestions
+from our OSS users. There's too much to list out since the previous release was
+cut, but I'll try to cover the highlights/important bits and promise to do
+better in the future :fingers_crossed:
+
+#### Major:
+
+* Complete rework of record validation with lenient mode support added to
+  octodns-dump so that data with validation problems can be dumped to config
+  files as a starting point. octoDNS now also ignores validation errors when
+  pulling the current state from a provider before planning changes. In both
+  cases this is best effort.
+* Naming of record keys are based on RFC-1035 and friends, previous names have
+  been kept for backwards compatibility until the 1.0 release.
+* Provider record type support is now explicit, i.e. opt-in, rather than
+  opt-out. This prevents bugs/oversights in record handling where providers
+  don't support (new) record types and didn't correctly ignore them.
+* ALIAS support for DNSimple, Dyn, NS1, PowerDNS
+* Ignored record support added, `octodns:\n  ignored: True`
+* Ns1Provider added
+
+#### Miscellaneous
+
+* Use a 3rd party lib for nautrual sorting of keys, rather than my old
+  implementation. Sorting can be disabled in the YamlProvider with
+  `enforce_order: False`.
+* Semi-colon/escaping fixes and improvements.
+* Meta record support, `TXT octodns-meta.<zone>`. For now just
+  `provider=<provider-id>`. Optionally turned on with `include_meta` manager
+  config val.
+* Validations check for CNAMEs co-existing with other records and error out if
+  found. Was a common mistaken/unknown issue and this surfaces the problem
+  early.
+* Sizeable refactor in the way Route53 record translation works to make it
+  cleaner/less hacky
+* Lots of docs type-o fixes
+* Fixed some pretty major bugs in DnsimpleProvider
+* Relax UnsafePlan checks a bit, more to come here
+* Set User-Agent header on Dyn health checks
+
+## v0.8.0 - 2017-03-14 - First public release

README.md

@@ -149,12 +149,15 @@ The above command pulled the existing data out of Route53 and placed the results
 
 | Provider | Record Support | GeoDNS Support | Notes |
 |--|--|--|--|
-| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CNAME, MX, NS, SPF, TXT | No | |
-| [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | |
+| [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | |
+| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted |
+| [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted |
 | [DynProvider](/octodns/provider/dyn.py) | All | Yes | |
+| [GoogleCloudProvider](/octodns/provider/googlecloud.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT  | No | |
 | [Ns1Provider](/octodns/provider/ns1.py) | All | No | |
+| [OVH](/octodns/provider/ovh.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | |
 | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | |
-| [Route53](/octodns/provider/route53.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | |
+| [Route53](/octodns/provider/route53.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | |
 | [TinyDNSSource](/octodns/source/tinydns.py) | A, CNAME, MX, NS, PTR | No | read-only |
 | [YamlProvider](/octodns/provider/yaml.py) | All | Yes | config |
 

octodns/__init__.py

@@ -1,8 +1,6 @@
-'''
-OctoDNS: DNS as code - Tools for managing DNS across multiple providers
-'''
+'OctoDNS: DNS as code - Tools for managing DNS across multiple providers'
 
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
-__VERSION__ = '0.8.0'
+__VERSION__ = '0.8.7'

octodns/cmds/dump.py

@@ -18,6 +18,9 @@ def main():
     parser.add_argument('--output-dir', required=True,
                         help='The directory into which the results will be '
                         'written (Note: will overwrite existing files)')
+    parser.add_argument('--lenient', action='store_true', default=False,
+                        help='Ignore record validations and do a best effort '
+                        'dump')
     parser.add_argument('zone', help='Zone to dump')
     parser.add_argument('source', nargs='+',
                         help='Source(s) to pull data from')
@@ -25,7 +28,7 @@ def main():
     args = parser.parse_args()
 
     manager = Manager(args.config_file)
-    manager.dump(args.zone, args.output_dir, *args.source)
+    manager.dump(args.zone, args.output_dir, args.lenient, *args.source)
 
 
 if __name__ == '__main__':

octodns/manager.py

@@ -6,13 +6,14 @@ from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
 from StringIO import StringIO
-from concurrent.futures import Future, ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor
 from importlib import import_module
 from os import environ
 import logging
 
-from .provider.base import BaseProvider
+from .provider.base import BaseProvider, Plan
 from .provider.yaml import YamlProvider
+from .record import Record
 from .yaml import safe_load
 from .zone import Zone
 
@@ -37,6 +38,17 @@ class _AggregateTarget(object):
         return True
 
 
+class MakeThreadFuture(object):
+
+    def __init__(self, func, args, kwargs):
+        self.func = func
+        self.args = args
+        self.kwargs = kwargs
+
+    def result(self):
+        return self.func(*self.args, **self.kwargs)
+
+
 class MainThreadExecutor(object):
     '''
     Dummy executor that runs things on the main thread during the involcation
@@ -47,19 +59,13 @@ class MainThreadExecutor(object):
     '''
 
     def submit(self, func, *args, **kwargs):
-        future = Future()
-        try:
-            future.set_result(func(*args, **kwargs))
-        except Exception as e:
-            # TODO: get right stacktrace here
-            future.set_exception(e)
-        return future
+        return MakeThreadFuture(func, args, kwargs)
 
 
 class Manager(object):
     log = logging.getLogger('Manager')
 
-    def __init__(self, config_file, max_workers=None):
+    def __init__(self, config_file, max_workers=None, include_meta=False):
         self.log.info('__init__: config_file=%s', config_file)
 
         # Read our config file
@@ -69,11 +75,16 @@ class Manager(object):
         manager_config = self.config.get('manager', {})
         max_workers = manager_config.get('max_workers', 1) \
             if max_workers is None else max_workers
+        self.log.info('__init__:   max_workers=%d', max_workers)
         if max_workers > 1:
             self._executor = ThreadPoolExecutor(max_workers=max_workers)
         else:
             self._executor = MainThreadExecutor()
 
+        self.include_meta = include_meta or manager_config.get('include_meta',
+                                                               False)
+        self.log.info('__init__:   max_workers=%s', self.include_meta)
+
         self.log.debug('__init__:   configuring providers')
         self.providers = {}
         for provider_name, provider_config in self.config['providers'].items():
@@ -175,6 +186,13 @@ class Manager(object):
         plans = []
 
         for target in targets:
+            if self.include_meta:
+                meta = Record.new(zone, 'octodns-meta', {
+                    'type': 'TXT',
+                    'ttl': 60,
+                    'value': 'provider={}'.format(target.id)
+                })
+                zone.add_record(meta, replace=True)
             plan = target.plan(zone)
             if plan:
                 plans.append((target, plan))
@@ -322,7 +340,7 @@ class Manager(object):
 
         return zb.changes(za, _AggregateTarget(a + b))
 
-    def dump(self, zone, output_dir, source, *sources):
+    def dump(self, zone, output_dir, lenient, source, *sources):
         '''
         Dump zone data from the specified source
         '''
@@ -341,9 +359,11 @@ class Manager(object):
 
         zone = Zone(zone, self.configured_sub_zones(zone))
         for source in sources:
-            source.populate(zone)
+            source.populate(zone, lenient=lenient)
 
         plan = target.plan(zone)
+        if plan is None:
+            plan = Plan(zone, zone, [])
         target.apply(plan)
 
     def validate_configs(self):

octodns/provider/azuredns.py

@@ -0,0 +1,443 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from azure.common.credentials import ServicePrincipalCredentials
+from azure.mgmt.dns import DnsManagementClient
+from msrestazure.azure_exceptions import CloudError
+
+from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \
+    SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone
+
+import logging
+from functools import reduce
+from ..record import Record
+from .base import BaseProvider
+
+
+class _AzureRecord(object):
+    '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls.
+
+        azuredns.py:
+        class: octodns.provider.azuredns._AzureRecord
+        An _AzureRecord is easily accessible to Azure DNS Management library
+        functions and is used to wrap all relevant data to create a record in
+        Azure.
+    '''
+    TYPE_MAP = {
+        'A': ARecord,
+        'AAAA': AaaaRecord,
+        'CNAME': CnameRecord,
+        'MX': MxRecord,
+        'SRV': SrvRecord,
+        'NS': NsRecord,
+        'PTR': PtrRecord,
+        'TXT': TxtRecord
+    }
+
+    def __init__(self, resource_group, record, delete=False):
+        '''Contructor for _AzureRecord.
+
+            Notes on Azure records: An Azure record set has the form
+            RecordSet(name=<...>, type=<...>, arecords=[...], aaaa_records, ..)
+            When constructing an azure record as done in self._apply_Create,
+            the argument parameters for an A record would be
+            parameters={'ttl': <int>, 'arecords': [ARecord(<str ip>),]}.
+            As another example for CNAME record:
+            parameters={'ttl': <int>, 'cname_record': CnameRecord(<str>)}.
+
+            Below, key_name and class_name are the dictionary key and Azure
+            Record class respectively.
+
+            :param resource_group: The name of resource group in Azure
+            :type  resource_group: str
+            :param record: An OctoDNS record
+            :type  record: ..record.Record
+            :param delete: If true, omit data parsing; not needed to delete
+            :type  delete: bool
+
+            :type return: _AzureRecord
+        '''
+        self.resource_group = resource_group
+        self.zone_name = record.zone.name[:len(record.zone.name) - 1]
+        self.relative_record_set_name = record.name or '@'
+        self.record_type = record._type
+
+        if delete:
+            return
+
+        # Refer to function docstring for key_name and class_name.
+        format_u_s = '' if record._type == 'A' else '_'
+        key_name = '{}{}records'.format(self.record_type, format_u_s).lower()
+        if record._type == 'CNAME':
+            key_name = key_name[:len(key_name) - 1]
+        azure_class = self.TYPE_MAP[self.record_type]
+
+        self.params = getattr(self, '_params_for_{}'.format(record._type))
+        self.params = self.params(record.data, key_name, azure_class)
+        self.params['ttl'] = record.ttl
+
+    def _params(self, data, key_name, azure_class):
+        try:
+            values = data['values']
+        except KeyError:
+            values = [data['value']]
+        return {key_name: [azure_class(v) for v in values]}
+
+    _params_for_A = _params
+    _params_for_AAAA = _params
+    _params_for_NS = _params
+    _params_for_PTR = _params
+
+    def _params_for_CNAME(self, data, key_name, azure_class):
+        return {key_name: azure_class(data['value'])}
+
+    def _params_for_MX(self, data, key_name, azure_class):
+        params = []
+        if 'values' in data:
+            for vals in data['values']:
+                params.append(azure_class(vals['preference'],
+                                          vals['exchange']))
+        else:  # Else there is a singular data point keyed by 'value'.
+            params.append(azure_class(data['value']['preference'],
+                                      data['value']['exchange']))
+        return {key_name: params}
+
+    def _params_for_SRV(self, data, key_name, azure_class):
+        params = []
+        if 'values' in data:
+            for vals in data['values']:
+                params.append(azure_class(vals['priority'],
+                                          vals['weight'],
+                                          vals['port'],
+                                          vals['target']))
+        else:  # Else there is a singular data point keyed by 'value'.
+            params.append(azure_class(data['value']['priority'],
+                                      data['value']['weight'],
+                                      data['value']['port'],
+                                      data['value']['target']))
+        return {key_name: params}
+
+    def _params_for_TXT(self, data, key_name, azure_class):
+        try:  # API for TxtRecord has list of str, even for singleton
+            values = data['values']
+        except KeyError:
+            values = [data['value']]
+        return {key_name: [azure_class([v]) for v in values]}
+
+    def _equals(self, b):
+        '''Checks whether two records are equal by comparing all fields.
+            :param b: Another _AzureRecord object
+            :type  b: _AzureRecord
+
+            :type return: bool
+        '''
+        def parse_dict(params):
+            vals = []
+            for char in params:
+                if char != 'ttl':
+                    list_records = params[char]
+                    try:
+                        for record in list_records:
+                            vals.append(record.__dict__)
+                    except:
+                        vals.append(list_records.__dict__)
+            vals.sort()
+            return vals
+
+        return (self.resource_group == b.resource_group) & \
+               (self.zone_name == b.zone_name) & \
+               (self.record_type == b.record_type) & \
+               (self.params['ttl'] == b.params['ttl']) & \
+               (parse_dict(self.params) == parse_dict(b.params)) & \
+               (self.relative_record_set_name == b.relative_record_set_name)
+
+    def __str__(self):
+        '''String representation of an _AzureRecord.
+            :type return: str
+        '''
+        string = 'Zone: {}; '.format(self.zone_name)
+        string += 'Name: {}; '.format(self.relative_record_set_name)
+        string += 'Type: {}; '.format(self.record_type)
+        if not hasattr(self, 'params'):
+            return string
+        string += 'Ttl: {}; '.format(self.params['ttl'])
+        for char in self.params:
+            if char != 'ttl':
+                try:
+                    for rec in self.params[char]:
+                        string += 'Record: {}; '.format(rec.__dict__)
+                except:
+                    string += 'Record: {}; '.format(self.params[char].__dict__)
+        return string
+
+
+def _check_endswith_dot(string):
+    return string if string.endswith('.') else string + '.'
+
+
+def _parse_azure_type(string):
+    '''Converts string representing an Azure RecordSet type to usual type.
+
+        :param string: the Azure type. eg: <Microsoft.Network/dnszones/A>
+        :type  string: str
+
+        :type return: str
+    '''
+    return string.split('/')[len(string.split('/')) - 1]
+
+
+class AzureProvider(BaseProvider):
+    '''
+    Azure DNS Provider
+
+    azuredns.py:
+        class: octodns.provider.azuredns.AzureProvider
+        # Current support of authentication of access to Azure services only
+        # includes using a Service Principal:
+        # https://docs.microsoft.com/en-us/azure/azure-resource-manager/
+        #                        resource-group-create-service-principal-portal
+        # The Azure Active Directory Application ID (aka client ID):
+        client_id:
+        # Authentication Key Value: (note this should be secret)
+        key:
+        # Directory ID (aka tenant ID):
+        directory_id:
+        # Subscription ID:
+        sub_id:
+        # Resource Group name:
+        resource_group:
+        # All are required to authenticate.
+
+        Example config file with variables:
+            "
+            ---
+            providers:
+              config:
+                class: octodns.provider.yaml.YamlProvider
+                directory: ./config (example path to directory of zone files)
+              azuredns:
+                class: octodns.provider.azuredns.AzureProvider
+                client_id: env/AZURE_APPLICATION_ID
+                key: env/AZURE_AUTHENICATION_KEY
+                directory_id: env/AZURE_DIRECTORY_ID
+                sub_id: env/AZURE_SUBSCRIPTION_ID
+                resource_group: 'TestResource1'
+
+            zones:
+              example.com.:
+                sources:
+                  - config
+                targets:
+                  - azuredns
+            "
+        The first four variables above can be hidden in environment variables
+        and octoDNS will automatically search for them in the shell. It is
+        possible to also hard-code into the config file: eg, resource_group.
+    '''
+    SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT'))
+
+    def __init__(self, id, client_id, key, directory_id, sub_id,
+                 resource_group, *args, **kwargs):
+        self.log = logging.getLogger('AzureProvider[{}]'.format(id))
+        self.log.debug('__init__: id=%s, client_id=%s, '
+                       'key=***, directory_id:%s', id, client_id, directory_id)
+        super(AzureProvider, self).__init__(id, *args, **kwargs)
+
+        credentials = ServicePrincipalCredentials(
+            client_id, secret=key, tenant=directory_id
+        )
+        self._dns_client = DnsManagementClient(credentials, sub_id)
+        self._resource_group = resource_group
+        self._azure_zones = set()
+
+    def _populate_zones(self):
+        self.log.debug('azure_zones: loading')
+        list_zones = self._dns_client.zones.list_by_resource_group
+        for zone in list_zones(self._resource_group):
+            self._azure_zones.add(zone.name)
+
+    def _check_zone(self, name, create=False):
+        '''Checks whether a zone specified in a source exist in Azure server.
+
+            Note that Azure zones omit end '.' eg: contoso.com vs contoso.com.
+            Returns the name if it exists.
+
+            :param name: Name of a zone to checks
+            :type  name: str
+            :param create: If True, creates the zone of that name.
+            :type  create: bool
+
+            :type return: str or None
+        '''
+        self.log.debug('_check_zone: name=%s', name)
+        try:
+            if name in self._azure_zones:
+                return name
+            self._dns_client.zones.get(self._resource_group, name)
+            self._azure_zones.add(name)
+            return name
+        except CloudError as err:
+            msg = 'The Resource \'Microsoft.Network/dnszones/{}\''.format(name)
+            msg += ' under resource group \'{}\''.format(self._resource_group)
+            msg += ' was not found.'
+            if msg == err.message:
+                # Then the only error is that the zone doesn't currently exist
+                if create:
+                    self.log.debug('_check_zone:no matching zone; creating %s',
+                                   name)
+                    create_zone = self._dns_client.zones.create_or_update
+                    create_zone(self._resource_group, name, Zone('global'))
+                    return name
+                else:
+                    return
+            raise
+
+    def populate(self, zone, target=False, lenient=False):
+        '''Required function of manager.py to collect records from zone.
+
+            Special notes for Azure.
+            Azure zone names omit final '.'
+            Azure root records names are represented by '@'. OctoDNS uses ''
+            Azure records created through online interface may have null values
+            (eg, no IP address for A record).
+            Azure online interface allows constructing records with null values
+            which are destroyed by _apply.
+
+            Specific quirks such as these are responsible for any non-obvious
+            parsing in this function and the functions '_params_for_*'.
+
+            :param zone: A dns zone
+            :type  zone: octodns.zone.Zone
+            :param target: Checks if Azure is source or target of config.
+                           Currently only supports as a target. Unused.
+            :type  target: bool
+            :param lenient: Unused. Check octodns.manager for usage.
+            :type  lenient: bool
+
+            :type return: void
+        '''
+        self.log.debug('populate: name=%s', zone.name)
+        before = len(zone.records)
+
+        zone_name = zone.name[:len(zone.name) - 1]
+        self._populate_zones()
+        self._check_zone(zone_name)
+
+        _records = set()
+        records = self._dns_client.record_sets.list_by_dns_zone
+        if self._check_zone(zone_name):
+            for azrecord in records(self._resource_group, zone_name):
+                if _parse_azure_type(azrecord.type) in self.SUPPORTS:
+                    _records.add(azrecord)
+            for azrecord in _records:
+                record_name = azrecord.name if azrecord.name != '@' else ''
+                typ = _parse_azure_type(azrecord.type)
+                data = getattr(self, '_data_for_{}'.format(typ))
+                data = data(azrecord)
+                data['type'] = typ
+                data['ttl'] = azrecord.ttl
+                record = Record.new(zone, record_name, data, source=self)
+                zone.add_record(record)
+
+        self.log.info('populate: found %s records', len(zone.records) - before)
+
+    def _data_for_A(self, azrecord):
+        return {'values': [ar.ipv4_address for ar in azrecord.arecords]}
+
+    def _data_for_AAAA(self, azrecord):
+        return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]}
+
+    def _data_for_CNAME(self, azrecord):
+        '''Parsing data from Azure DNS Client record call
+            :param azrecord: a return of a call to list azure records
+            :type  azrecord: azure.mgmt.dns.models.RecordSet
+
+            :type  return: dict
+
+            CNAME and PTR both use the catch block to catch possible empty
+            records. Refer to population comment.
+        '''
+        try:
+            return {'value': _check_endswith_dot(azrecord.cname_record.cname)}
+        except:
+            return {'value': '.'}
+
+    def _data_for_MX(self, azrecord):
+        return {'values': [{'preference': ar.preference,
+                            'exchange': ar.exchange}
+                           for ar in azrecord.mx_records]}
+
+    def _data_for_NS(self, azrecord):
+        vals = [ar.nsdname for ar in azrecord.ns_records]
+        return {'values': [_check_endswith_dot(val) for val in vals]}
+
+    def _data_for_PTR(self, azrecord):
+        try:
+            ptrdname = azrecord.ptr_records[0].ptrdname
+            return {'value': _check_endswith_dot(ptrdname)}
+        except:
+            return {'value': '.'}
+
+    def _data_for_SRV(self, azrecord):
+        return {'values': [{'priority': ar.priority, 'weight': ar.weight,
+                            'port': ar.port, 'target': ar.target}
+                           for ar in azrecord.srv_records]}
+
+    def _data_for_TXT(self, azrecord):
+        return {'values': [reduce((lambda a, b: a + b), ar.value)
+                           for ar in azrecord.txt_records]}
+
+    def _apply_Create(self, change):
+        '''A record from change must be created.
+
+            :param change: a change object
+            :type  change: octodns.record.Change
+
+            :type return: void
+        '''
+        ar = _AzureRecord(self._resource_group, change.new)
+        create = self._dns_client.record_sets.create_or_update
+
+        create(resource_group_name=ar.resource_group,
+               zone_name=ar.zone_name,
+               relative_record_set_name=ar.relative_record_set_name,
+               record_type=ar.record_type,
+               parameters=ar.params)
+
+        self.log.debug('*  Success Create/Update: {}'.format(ar))
+
+    _apply_Update = _apply_Create
+
+    def _apply_Delete(self, change):
+        ar = _AzureRecord(self._resource_group, change.existing, delete=True)
+        delete = self._dns_client.record_sets.delete
+
+        delete(self._resource_group, ar.zone_name, ar.relative_record_set_name,
+               ar.record_type)
+
+        self.log.debug('*  Success Delete: {}'.format(ar))
+
+    def _apply(self, plan):
+        '''Required function of manager.py to actually apply a record change.
+
+            :param plan: Contains the zones and changes to be made
+            :type  plan: octodns.provider.base.Plan
+
+            :type return: void
+        '''
+        desired = plan.desired
+        changes = plan.changes
+        self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name,
+                       len(changes))
+
+        azure_zone_name = desired.name[:len(desired.name) - 1]
+        self._check_zone(azure_zone_name, create=True)
+
+        for change in changes:
+            class_name = change.__class__.__name__
+            getattr(self, '_apply_{}'.format(class_name))(change)

octodns/provider/base.py

@@ -21,10 +21,14 @@ class Plan(object):
     MAX_SAFE_DELETE_PCENT = .3
     MIN_EXISTING_RECORDS = 10
 
-    def __init__(self, existing, desired, changes):
+    def __init__(self, existing, desired, changes,
+                 update_pcent_threshold=MAX_SAFE_UPDATE_PCENT,
+                 delete_pcent_threshold=MAX_SAFE_DELETE_PCENT):
         self.existing = existing
         self.desired = desired
         self.changes = changes
+        self.update_pcent_threshold = update_pcent_threshold
+        self.delete_pcent_threshold = delete_pcent_threshold
 
         change_counts = {
             'Create': 0,
@@ -55,20 +59,20 @@ class Plan(object):
             update_pcent = self.change_counts['Update'] / existing_record_count
             delete_pcent = self.change_counts['Delete'] / existing_record_count
 
-            if update_pcent > self.MAX_SAFE_UPDATE_PCENT:
-                raise UnsafePlan('Too many updates, %s is over %s percent'
-                                 '(%s/%s)',
-                                 update_pcent,
-                                 self.MAX_SAFE_UPDATE_PCENT * 100,
-                                 self.change_counts['Update'],
-                                 existing_record_count)
-            if delete_pcent > self.MAX_SAFE_DELETE_PCENT:
-                raise UnsafePlan('Too many deletes, %s is over %s percent'
-                                 '(%s/%s)',
-                                 delete_pcent,
-                                 self.MAX_SAFE_DELETE_PCENT * 100,
-                                 self.change_counts['Delete'],
-                                 existing_record_count)
+            if update_pcent > self.update_pcent_threshold:
+                raise UnsafePlan('Too many updates, {} is over {} percent'
+                                 '({}/{})'.format(
+                                     update_pcent,
+                                     self.MAX_SAFE_UPDATE_PCENT * 100,
+                                     self.change_counts['Update'],
+                                     existing_record_count))
+            if delete_pcent > self.delete_pcent_threshold:
+                raise UnsafePlan('Too many deletes, {} is over {} percent'
+                                 '({}/{})'.format(
+                                     delete_pcent,
+                                     self.MAX_SAFE_DELETE_PCENT * 100,
+                                     self.change_counts['Delete'],
+                                     existing_record_count))
 
     def __repr__(self):
         return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \
@@ -79,11 +83,19 @@ class Plan(object):
 
 class BaseProvider(BaseSource):
 
-    def __init__(self, id, apply_disabled=False):
+    def __init__(self, id, apply_disabled=False,
+                 update_pcent_threshold=Plan.MAX_SAFE_UPDATE_PCENT,
+                 delete_pcent_threshold=Plan.MAX_SAFE_DELETE_PCENT):
         super(BaseProvider, self).__init__(id)
-        self.log.debug('__init__: id=%s, apply_disabled=%s', id,
-                       apply_disabled)
+        self.log.debug('__init__: id=%s, apply_disabled=%s, '
+                       'update_pcent_threshold=%d, delete_pcent_threshold=%d',
+                       id,
+                       apply_disabled,
+                       update_pcent_threshold,
+                       delete_pcent_threshold)
         self.apply_disabled = apply_disabled
+        self.update_pcent_threshold = update_pcent_threshold
+        self.delete_pcent_threshold = delete_pcent_threshold
 
     def _include_change(self, change):
         '''
@@ -104,7 +116,7 @@ class BaseProvider(BaseSource):
         self.log.info('plan: desired=%s', desired.name)
 
         existing = Zone(desired.name, desired.sub_zones)
-        self.populate(existing, target=True)
+        self.populate(existing, target=True, lenient=True)
 
         # compute the changes at the zone/record level
         changes = existing.changes(desired, self)
@@ -124,7 +136,9 @@ class BaseProvider(BaseSource):
             changes += extra
 
         if changes:
-            plan = Plan(existing, desired, changes)
+            plan = Plan(existing, desired, changes,
+                        self.update_pcent_threshold,
+                        self.delete_pcent_threshold)
             self.log.info('plan:   %s', plan)
             return plan
         self.log.info('plan:   No changes')

octodns/provider/cloudflare.py

@@ -36,7 +36,7 @@ class CloudflareProvider(BaseProvider):
     '''
     SUPPORTS_GEO = False
     # TODO: support SRV
-    UNSUPPORTED_TYPES = ('ALIAS', 'NAPTR', 'PTR', 'SOA', 'SRV', 'SSHFP')
+    SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SPF', 'TXT'))
 
     MIN_TTL = 120
     TIMEOUT = 15
@@ -56,9 +56,6 @@ class CloudflareProvider(BaseProvider):
         self._zones = None
         self._zone_records = {}
 
-    def supports(self, record):
-        return record._type not in self.UNSUPPORTED_TYPES
-
     def _request(self, method, path, params=None, data=None):
         self.log.debug('_request: method=%s, path=%s', method, path)
 
@@ -107,6 +104,17 @@ class CloudflareProvider(BaseProvider):
             'values': [r['content'].replace(';', '\;') for r in records],
         }
 
+    def _data_for_CAA(self, _type, records):
+        values = []
+        for r in records:
+            data = r['data']
+            values.append(data)
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values,
+        }
+
     def _data_for_CNAME(self, _type, records):
         only = records[0]
         return {
@@ -119,8 +127,8 @@ class CloudflareProvider(BaseProvider):
         values = []
         for r in records:
             values.append({
-                'priority': r['priority'],
-                'value': '{}.'.format(r['content']),
+                'preference': r['priority'],
+                'exchange': '{}.'.format(r['content']),
             })
         return {
             'ttl': records[0]['ttl'],
@@ -157,8 +165,9 @@ class CloudflareProvider(BaseProvider):
 
         return self._zone_records[zone.name]
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: name=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
 
         before = len(zone.records)
         records = self.zone_records(zone)
@@ -167,14 +176,15 @@ class CloudflareProvider(BaseProvider):
             for record in records:
                 name = zone.hostname_from_fqdn(record['name'])
                 _type = record['type']
-                if _type not in self.UNSUPPORTED_TYPES:
+                if _type in self.SUPPORTS:
                     values[name][record['type']].append(record)
 
             for name, types in values.items():
                 for _type, records in types.items():
                     data_for = getattr(self, '_data_for_{}'.format(_type))
                     data = data_for(_type, records)
-                    record = Record.new(zone, name, data, source=self)
+                    record = Record.new(zone, name, data, source=self,
+                                        lenient=lenient)
                     zone.add_record(record)
 
         self.log.info('populate:   found %s records',
@@ -198,6 +208,16 @@ class CloudflareProvider(BaseProvider):
     _contents_for_NS = _contents_for_multiple
     _contents_for_SPF = _contents_for_multiple
 
+    def _contents_for_CAA(self, record):
+        for value in record.values:
+            yield {
+                'data': {
+                    'flags': value.flags,
+                    'tag': value.tag,
+                    'value': value.value,
+                }
+            }
+
     def _contents_for_TXT(self, record):
         for value in record.values:
             yield {'content': value.replace('\;', ';')}
@@ -208,8 +228,8 @@ class CloudflareProvider(BaseProvider):
     def _contents_for_MX(self, record):
         for value in record.values:
             yield {
-                'priority': value.priority,
-                'content': value.value
+                'priority': value.preference,
+                'content': value.exchange
             }
 
     def _apply_Create(self, change):

octodns/provider/dnsimple.py

@@ -91,6 +91,8 @@ class DnsimpleProvider(BaseProvider):
         account: 42
     '''
     SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS',
+                    'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT'))
 
     def __init__(self, id, token, account, *args, **kwargs):
         self.log = logging.getLogger('DnsimpleProvider[{}]'.format(id))
@@ -112,6 +114,21 @@ class DnsimpleProvider(BaseProvider):
     _data_for_SPF = _data_for_multiple
     _data_for_TXT = _data_for_multiple
 
+    def _data_for_CAA(self, _type, records):
+        values = []
+        for record in records:
+            flags, tag, value = record['content'].split(' ')
+            values.append({
+                'flags': flags,
+                'tag': tag,
+                'value': value[1:-1],
+            })
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values
+        }
+
     def _data_for_CNAME(self, _type, records):
         record = records[0]
         return {
@@ -126,8 +143,8 @@ class DnsimpleProvider(BaseProvider):
         values = []
         for record in records:
             values.append({
-                'priority': record['priority'],
-                'value': '{}.'.format(record['content'])
+                'preference': record['priority'],
+                'exchange': '{}.'.format(record['content'])
             })
         return {
             'ttl': records[0]['ttl'],
@@ -232,8 +249,9 @@ class DnsimpleProvider(BaseProvider):
 
         return self._zone_records[zone.name]
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: name=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
 
         values = defaultdict(lambda: defaultdict(list))
         for record in self.zone_records(zone):
@@ -250,7 +268,8 @@ class DnsimpleProvider(BaseProvider):
         for name, types in values.items():
             for _type, records in types.items():
                 data_for = getattr(self, '_data_for_{}'.format(_type))
-                record = Record.new(zone, name, data_for(_type, records))
+                record = Record.new(zone, name, data_for(_type, records),
+                                    source=self, lenient=lenient)
                 zone.add_record(record)
 
         self.log.info('populate:   found %s records',
@@ -271,6 +290,16 @@ class DnsimpleProvider(BaseProvider):
     _params_for_SPF = _params_for_multiple
     _params_for_TXT = _params_for_multiple
 
+    def _params_for_CAA(self, record):
+        for value in record.values:
+            yield {
+                'content': '{} {} "{}"'.format(value.flags, value.tag,
+                                               value.value),
+                'name': record.name,
+                'ttl': record.ttl,
+                'type': record._type
+            }
+
     def _params_for_single(self, record):
         yield {
             'content': record.value,
@@ -286,9 +315,9 @@ class DnsimpleProvider(BaseProvider):
     def _params_for_MX(self, record):
         for value in record.values:
             yield {
-                'content': value.value,
+                'content': value.exchange,
                 'name': record.name,
-                'priority': value.priority,
+                'priority': value.preference,
                 'ttl': record.ttl,
                 'type': record._type
             }

octodns/provider/dyn.py

@@ -157,10 +157,12 @@ class DynProvider(BaseProvider):
     than one account active at a time. See DynProvider._check_dyn_sess for some
     related bits.
     '''
+
     RECORDS_TO_TYPE = {
         'a_records': 'A',
         'aaaa_records': 'AAAA',
         'alias_records': 'ALIAS',
+        'caa_records': 'CAA',
         'cname_records': 'CNAME',
         'mx_records': 'MX',
         'naptr_records': 'NAPTR',
@@ -172,6 +174,7 @@ class DynProvider(BaseProvider):
         'txt_records': 'TXT',
     }
     TYPE_TO_RECORDS = {v: k for k, v in RECORDS_TO_TYPE.items()}
+    SUPPORTS = set(TYPE_TO_RECORDS.keys())
 
     # https://help.dyn.com/predefined-geotm-regions-groups/
     REGION_CODES = {
@@ -247,6 +250,14 @@ class DynProvider(BaseProvider):
             'value': record.alias
         }
 
+    def _data_for_CAA(self, _type, records):
+        return {
+            'type': _type,
+            'ttl': records[0].ttl,
+            'values': [{'flags': r.flags, 'tag': r.tag, 'value': r.value}
+                       for r in records],
+        }
+
     def _data_for_CNAME(self, _type, records):
         record = records[0]
         return {
@@ -259,7 +270,7 @@ class DynProvider(BaseProvider):
         return {
             'type': _type,
             'ttl': records[0].ttl,
-            'values': [{'priority': r.preference, 'value': r.exchange}
+            'values': [{'preference': r.preference, 'exchange': r.exchange}
                        for r in records],
         }
 
@@ -334,7 +345,9 @@ class DynProvider(BaseProvider):
             for td in get_all_dsf_services():
                 try:
                     fqdn, _type = td.label.split(':', 1)
-                except ValueError:
+                except ValueError as e:
+                    self.log.warn("Failed to load TraficDirector '%s': %s",
+                                  td.label, e.message)
                     continue
                 tds[fqdn][_type] = td
             self._traffic_directors = dict(tds)
@@ -391,8 +404,10 @@ class DynProvider(BaseProvider):
 
         return td_records
 
-    def populate(self, zone, target=False):
-        self.log.info('populate: zone=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+
         before = len(zone.records)
 
         self._check_dyn_sess()
@@ -417,7 +432,8 @@ class DynProvider(BaseProvider):
                 for _type, records in types.items():
                     data_for = getattr(self, '_data_for_{}'.format(_type))
                     data = data_for(_type, records)
-                    record = Record.new(zone, name, data, source=self)
+                    record = Record.new(zone, name, data, source=self,
+                                        lenient=lenient)
                     if record not in td_records:
                         zone.add_record(record)
 
@@ -460,6 +476,13 @@ class DynProvider(BaseProvider):
 
     _kwargs_for_AAAA = _kwargs_for_A
 
+    def _kwargs_for_CAA(self, record):
+        return [{
+            'flags': v.flags,
+            'tag': v.tag,
+            'value': v.value,
+        } for v in record.values]
+
     def _kwargs_for_CNAME(self, record):
         return [{
             'cname': record.value,
@@ -478,8 +501,8 @@ class DynProvider(BaseProvider):
 
     def _kwargs_for_MX(self, record):
         return [{
-            'preference': v.priority,
-            'exchange': v.value,
+            'preference': v.preference,
+            'exchange': v.exchange,
             'ttl': record.ttl,
         } for v in record.values]
 
@@ -517,7 +540,7 @@ class DynProvider(BaseProvider):
         return [{
             'txtdata': v,
             'ttl': record.ttl,
-        } for v in record.values]
+        } for v in record.chunked_values]
 
     def _kwargs_for_SRV(self, record):
         return [{

octodns/provider/googlecloud.py

@@ -0,0 +1,333 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+import shlex
+import time
+from logging import getLogger
+from uuid import uuid4
+
+from google.cloud import dns
+
+from .base import BaseProvider
+from ..record import Record
+
+
+class GoogleCloudProvider(BaseProvider):
+    """
+    Google Cloud DNS provider
+
+    google_cloud:
+        class: octodns.provider.googlecloud.GoogleCloudProvider
+        # Credentials file for a service_account or other account can be
+        # specified with the GOOGLE_APPLICATION_CREDENTIALS environment
+        # variable. (https://console.cloud.google.com/apis/credentials)
+        #
+        # The project to work on (not required)
+        # project: foobar
+        #
+        # The File with the google credentials (not required). If used, the
+        # "project" parameter needs to be set, else it will fall back to the
+        #  "default credentials"
+        # credentials_file: ~/google_cloud_credentials_file.json
+        #
+    """
+
+    SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NAPTR',
+                    'NS', 'PTR', 'SPF', 'SRV', 'TXT'))
+    SUPPORTS_GEO = False
+
+    CHANGE_LOOP_WAIT = 5
+
+    def __init__(self, id, project=None, credentials_file=None,
+                 *args, **kwargs):
+
+        if credentials_file:
+            self.gcloud_client = dns.Client.from_service_account_json(
+                credentials_file, project=project)
+        else:
+            self.gcloud_client = dns.Client(project=project)
+
+        # Logger
+        self.log = getLogger('GoogleCloudProvider[{}]'.format(id))
+        self.id = id
+
+        self._gcloud_zones = {}
+
+        super(GoogleCloudProvider, self).__init__(id, *args, **kwargs)
+
+    def _apply(self, plan):
+        """Required function of manager.py to actually apply a record change.
+
+            :param plan: Contains the zones and changes to be made
+            :type  plan: octodns.provider.base.Plan
+
+            :type return: void
+        """
+        desired = plan.desired
+        changes = plan.changes
+
+        self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name,
+                       len(changes))
+
+        # Get gcloud zone, or create one if none existed before.
+        if desired.name not in self.gcloud_zones:
+            gcloud_zone = self._create_gcloud_zone(desired.name)
+        else:
+            gcloud_zone = self.gcloud_zones.get(desired.name)
+
+        gcloud_changes = gcloud_zone.changes()
+
+        for change in changes:
+            class_name = change.__class__.__name__
+            _rrset_func = getattr(
+                self, '_rrset_for_{}'.format(change.record._type))
+
+            if class_name == 'Create':
+                gcloud_changes.add_record_set(
+                    _rrset_func(gcloud_zone, change.record))
+            elif class_name == 'Delete':
+                gcloud_changes.delete_record_set(
+                    _rrset_func(gcloud_zone, change.record))
+            elif class_name == 'Update':
+                gcloud_changes.delete_record_set(
+                    _rrset_func(gcloud_zone, change.existing))
+                gcloud_changes.add_record_set(
+                    _rrset_func(gcloud_zone, change.new))
+            else:
+                raise RuntimeError('Change type "{}" for change "{!s}" '
+                                   'is none of "Create", "Delete" or "Update'
+                                   .format(class_name, change))
+
+        gcloud_changes.create()
+
+        for i in range(120):
+            gcloud_changes.reload()
+            # https://cloud.google.com/dns/api/v1/changes#resource
+            # status can be one of either "pending" or "done"
+            if gcloud_changes.status != 'pending':
+                break
+            self.log.debug("Waiting for changes to complete")
+            time.sleep(self.CHANGE_LOOP_WAIT)
+
+        if gcloud_changes.status != 'done':
+            raise RuntimeError("Timeout reached after {} seconds".format(
+                i * self.CHANGE_LOOP_WAIT))
+
+    def _create_gcloud_zone(self, dns_name):
+        """Creates a google cloud ManagedZone with dns_name, and zone named
+            derived from it. calls .create() method and returns it.
+
+            :param dns_name: fqdn of zone to create
+            :type  dns_name: str
+
+            :type return: new google.cloud.dns.ManagedZone
+        """
+        # Zone name must begin with a letter, end with a letter or digit,
+        # and only contain lowercase letters, digits or dashes
+        zone_name = '{}-{}'.format(
+            dns_name[:-1].replace('.', '-'), uuid4().hex)
+
+        gcloud_zone = self.gcloud_client.zone(
+            name=zone_name,
+            dns_name=dns_name
+        )
+        gcloud_zone.create(client=self.gcloud_client)
+
+        # add this new zone to the list of zones.
+        self._gcloud_zones[gcloud_zone.dns_name] = gcloud_zone
+
+        self.log.info("Created zone {}. Fqdn {}.".format(zone_name, dns_name))
+
+        return gcloud_zone
+
+    def _get_gcloud_records(self, gcloud_zone, page_token=None):
+        """ Generator function which yields ResourceRecordSet for the managed
+            gcloud zone, until there are no more records to pull.
+
+            :param gcloud_zone: zone to pull records from
+            :type gcloud_zone: google.cloud.dns.ManagedZone
+            :param page_token: page token for the page to get
+
+            :return: a resource record set
+            :type return: google.cloud.dns.ResourceRecordSet
+        """
+        gcloud_iterator = gcloud_zone.list_resource_record_sets(
+            page_token=page_token)
+        for gcloud_record in gcloud_iterator:
+            yield gcloud_record
+        # This is to get results which may be on a "paged" page.
+        # (if more than max_results) entries.
+        if gcloud_iterator.next_page_token:
+            for gcloud_record in self._get_gcloud_records(
+                    gcloud_zone, gcloud_iterator.next_page_token):
+                # yield from is in python 3 only.
+                yield gcloud_record
+
+    def _get_cloud_zones(self, page_token=None):
+        """Load all ManagedZones into the self._gcloud_zones dict which is
+        mapped with the dns_name as key.
+
+        :return: void
+        """
+
+        gcloud_zones = self.gcloud_client.list_zones(page_token=page_token)
+        for gcloud_zone in gcloud_zones:
+            self._gcloud_zones[gcloud_zone.dns_name] = gcloud_zone
+
+        if gcloud_zones.next_page_token:
+            self._get_cloud_zones(gcloud_zones.next_page_token)
+
+    @property
+    def gcloud_zones(self):
+        if not self._gcloud_zones:
+            self._get_cloud_zones()
+        return self._gcloud_zones
+
+    def populate(self, zone, target=False, lenient=False):
+        """Required function of manager.py to collect records from zone.
+
+            :param zone: A dns zone
+            :type  zone: octodns.zone.Zone
+            :param target: Unused.
+            :type  target: bool
+            :param lenient: Unused. Check octodns.manager for usage.
+            :type  lenient: bool
+
+            :type return: void
+        """
+
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+        before = len(zone.records)
+
+        gcloud_zone = self.gcloud_zones.get(zone.name)
+
+        if gcloud_zone:
+            for gcloud_record in self._get_gcloud_records(gcloud_zone):
+                if gcloud_record.record_type.upper() not in self.SUPPORTS:
+                    continue
+
+                record_name = gcloud_record.name
+                if record_name.endswith(zone.name):
+                    # google cloud always return fqdn. Make relative record
+                    # here. "root" records will then get the '' record_name,
+                    # which is also the way octodns likes it.
+                    record_name = record_name[:-(len(zone.name) + 1)]
+                typ = gcloud_record.record_type.upper()
+                data = getattr(self, '_data_for_{}'.format(typ))
+                data = data(gcloud_record)
+                data['type'] = typ
+                data['ttl'] = gcloud_record.ttl
+                self.log.debug('populate: adding record {} records: {!s}'
+                               .format(record_name, data))
+                record = Record.new(zone, record_name, data, source=self)
+                zone.add_record(record)
+
+        self.log.info('populate: found %s records', len(zone.records) - before)
+
+    def _data_for_A(self, gcloud_record):
+        return {
+            'values': gcloud_record.rrdatas
+        }
+
+    _data_for_AAAA = _data_for_A
+
+    def _data_for_CAA(self, gcloud_record):
+        return {
+            'values': [{
+                'flags': v[0],
+                'tag': v[1],
+                'value': v[2]}
+                for v in [shlex.split(g) for g in gcloud_record.rrdatas]]}
+
+    def _data_for_CNAME(self, gcloud_record):
+        return {
+            'value': gcloud_record.rrdatas[0]
+        }
+
+    def _data_for_MX(self, gcloud_record):
+        return {'values': [{
+            "preference": v[0],
+            "exchange": v[1]}
+            for v in [shlex.split(g) for g in gcloud_record.rrdatas]]}
+
+    def _data_for_NAPTR(self, gcloud_record):
+        return {'values': [{
+            'order': v[0],
+            'preference': v[1],
+            'flags': v[2],
+            'service': v[3],
+            'regexp': v[4],
+            'replacement': v[5]}
+            for v in [shlex.split(g) for g in gcloud_record.rrdatas]]}
+
+    _data_for_NS = _data_for_A
+
+    _data_for_PTR = _data_for_CNAME
+
+    def _data_for_SPF(self, gcloud_record):
+        if len(gcloud_record.rrdatas) > 1:
+            return {
+                'values': gcloud_record.rrdatas}
+        return {
+            'value': gcloud_record.rrdatas[0]}
+
+    def _data_for_SRV(self, gcloud_record):
+        return {'values': [{
+            'priority': v[0],
+            'weight': v[1],
+            'port': v[2],
+            'target': v[3]}
+            for v in [shlex.split(g) for g in gcloud_record.rrdatas]]}
+
+    _data_for_TXT = _data_for_SPF
+
+    def _rrset_for_A(self, gcloud_zone, record):
+        return gcloud_zone.resource_record_set(
+            record.fqdn, record._type, record.ttl, record.values)
+
+    _rrset_for_AAAA = _rrset_for_A
+
+    def _rrset_for_CAA(self, gcloud_zone, record):
+        return gcloud_zone.resource_record_set(
+            record.fqdn, record._type, record.ttl, [
+                '{} {} {}'.format(v.flags, v.tag, v.value)
+                for v in record.values])
+
+    def _rrset_for_CNAME(self, gcloud_zone, record):
+        return gcloud_zone.resource_record_set(
+            record.fqdn, record._type, record.ttl, [record.value])
+
+    def _rrset_for_MX(self, gcloud_zone, record):
+        return gcloud_zone.resource_record_set(
+            record.fqdn, record._type, record.ttl, [
+                '{} {}'.format(v.preference, v.exchange)
+                for v in record.values])
+
+    def _rrset_for_NAPTR(self, gcloud_zone, record):
+        return gcloud_zone.resource_record_set(
+            record.fqdn, record._type, record.ttl, [
+                '{} {} "{}" "{}" "{}" {}'.format(
+                    v.order, v.preference, v.flags, v.service,
+                    v.regexp, v.replacement) for v in record.values])
+
+    _rrset_for_NS = _rrset_for_A
+
+    _rrset_for_PTR = _rrset_for_CNAME
+
+    def _rrset_for_SPF(self, gcloud_zone, record):
+        return gcloud_zone.resource_record_set(
+            record.fqdn, record._type, record.ttl, record.chunked_values)
+
+    def _rrset_for_SRV(self, gcloud_zone, record):
+        return gcloud_zone.resource_record_set(
+            record.fqdn, record._type, record.ttl, [
+                '{} {} {} {}'
+                .format(v.priority, v.weight, v.port, v.target)
+                for v in record.values])
+
+    _rrset_for_TXT = _rrset_for_SPF

octodns/provider/ns1.py

@@ -7,7 +7,8 @@ from __future__ import absolute_import, division, print_function, \
 
 from logging import getLogger
 from nsone import NSONE
-from nsone.rest.errors import ResourceException
+from nsone.rest.errors import RateLimitException, ResourceException
+from time import sleep
 
 from ..record import Record
 from .base import BaseProvider
@@ -22,6 +23,9 @@ class Ns1Provider(BaseProvider):
         api_key: env/NS1_API_KEY
     '''
     SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS',
+                    'PTR', 'SPF', 'SRV', 'TXT'))
+
     ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found'
 
     def __init__(self, id, api_key, *args, **kwargs):
@@ -30,9 +34,6 @@ class Ns1Provider(BaseProvider):
         super(Ns1Provider, self).__init__(id, *args, **kwargs)
         self._client = NSONE(apiKey=api_key)
 
-    def supports(self, record):
-        return record._type != 'SSHFP'
-
     def _data_for_A(self, _type, record):
         return {
             'ttl': record['ttl'],
@@ -41,8 +42,31 @@ class Ns1Provider(BaseProvider):
         }
 
     _data_for_AAAA = _data_for_A
-    _data_for_SPF = _data_for_A
-    _data_for_TXT = _data_for_A
+
+    def _data_for_SPF(self, _type, record):
+        values = [v.replace(';', '\;') for v in record['short_answers']]
+        return {
+            'ttl': record['ttl'],
+            'type': _type,
+            'values': values
+        }
+
+    _data_for_TXT = _data_for_SPF
+
+    def _data_for_CAA(self, _type, record):
+        values = []
+        for answer in record['short_answers']:
+            flags, tag, value = answer.split(' ', 2)
+            values.append({
+                'flags': flags,
+                'tag': tag,
+                'value': value,
+            })
+        return {
+            'ttl': record['ttl'],
+            'type': _type,
+            'values': values,
+        }
 
     def _data_for_CNAME(self, _type, record):
         return {
@@ -57,10 +81,10 @@ class Ns1Provider(BaseProvider):
     def _data_for_MX(self, _type, record):
         values = []
         for answer in record['short_answers']:
-            priority, value = answer.split(' ', 1)
+            preference, exchange = answer.split(' ', 1)
             values.append({
-                'priority': priority,
-                'value': value,
+                'preference': preference,
+                'exchange': exchange,
             })
         return {
             'ttl': record['ttl'],
@@ -111,8 +135,9 @@ class Ns1Provider(BaseProvider):
             'values': values,
         }
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: name=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
 
         try:
             nsone_zone = self._client.loadZone(zone.name[:-1])
@@ -127,7 +152,8 @@ class Ns1Provider(BaseProvider):
             _type = record['type']
             data_for = getattr(self, '_data_for_{}'.format(_type))
             name = zone.hostname_from_fqdn(record['domain'])
-            record = Record.new(zone, name, data_for(_type, record))
+            record = Record.new(zone, name, data_for(_type, record),
+                                source=self, lenient=lenient)
             zone.add_record(record)
 
         self.log.info('populate:   found %s records',
@@ -138,8 +164,19 @@ class Ns1Provider(BaseProvider):
 
     _params_for_AAAA = _params_for_A
     _params_for_NS = _params_for_A
-    _params_for_SPF = _params_for_A
-    _params_for_TXT = _params_for_A
+
+    def _params_for_SPF(self, record):
+        # NS1 seems to be the only provider that doesn't want things escaped in
+        # values so we have to strip them here and add them when going the
+        # other way
+        values = [v.replace('\;', ';') for v in record.values]
+        return {'answers': values, 'ttl': record.ttl}
+
+    _params_for_TXT = _params_for_SPF
+
+    def _params_for_CAA(self, record):
+        values = [(v.flags, v.tag, v.value) for v in record.values]
+        return {'answers': values, 'ttl': record.ttl}
 
     def _params_for_CNAME(self, record):
         return {'answers': [record.value], 'ttl': record.ttl}
@@ -148,7 +185,7 @@ class Ns1Provider(BaseProvider):
     _params_for_PTR = _params_for_CNAME
 
     def _params_for_MX(self, record):
-        values = [(v.priority, v.value) for v in record.values]
+        values = [(v.preference, v.exchange) for v in record.values]
         return {'answers': values, 'ttl': record.ttl}
 
     def _params_for_NAPTR(self, record):
@@ -169,7 +206,14 @@ class Ns1Provider(BaseProvider):
         name = self._get_name(new)
         _type = new._type
         params = getattr(self, '_params_for_{}'.format(_type))(new)
-        getattr(nsone_zone, 'add_{}'.format(_type))(name, **params)
+        meth = getattr(nsone_zone, 'add_{}'.format(_type))
+        try:
+            meth(name, **params)
+        except RateLimitException as e:
+            self.log.warn('_apply_Create: rate limit encountered, pausing '
+                          'for %ds and trying again', e.period)
+            sleep(e.period)
+            meth(name, **params)
 
     def _apply_Update(self, nsone_zone, change):
         existing = change.existing
@@ -178,14 +222,26 @@ class Ns1Provider(BaseProvider):
         record = nsone_zone.loadRecord(name, _type)
         new = change.new
         params = getattr(self, '_params_for_{}'.format(_type))(new)
-        record.update(**params)
+        try:
+            record.update(**params)
+        except RateLimitException as e:
+            self.log.warn('_apply_Update: rate limit encountered, pausing '
+                          'for %ds and trying again', e.period)
+            sleep(e.period)
+            record.update(**params)
 
     def _apply_Delete(self, nsone_zone, change):
         existing = change.existing
         name = self._get_name(existing)
         _type = existing._type
         record = nsone_zone.loadRecord(name, _type)
-        record.delete()
+        try:
+            record.delete()
+        except RateLimitException as e:
+            self.log.warn('_apply_Delete: rate limit encountered, pausing '
+                          'for %ds and trying again', e.period)
+            sleep(e.period)
+            record.delete()
 
     def _apply(self, plan):
         desired = plan.desired

octodns/provider/ovh.py

@@ -0,0 +1,322 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+import logging
+from collections import defaultdict
+
+import ovh
+
+from octodns.record import Record
+from .base import BaseProvider
+
+
+class OvhProvider(BaseProvider):
+    """
+    OVH provider using API v6
+
+    ovh:
+        class: octodns.provider.ovh.OvhProvider
+        # OVH api v6 endpoint
+        endpoint: ovh-eu
+        # API application key
+        application_key: 1234
+        # API application secret
+        application_secret: 1234
+        # API consumer key
+        consumer_key: 1234
+    """
+
+    SUPPORTS_GEO = False
+
+    SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', 'SPF',
+                    'SRV', 'SSHFP', 'TXT'))
+
+    def __init__(self, id, endpoint, application_key, application_secret,
+                 consumer_key, *args, **kwargs):
+        self.log = logging.getLogger('OvhProvider[{}]'.format(id))
+        self.log.debug('__init__: id=%s, endpoint=%s, application_key=%s, '
+                       'application_secret=***, consumer_key=%s', id, endpoint,
+                       application_key, consumer_key)
+        super(OvhProvider, self).__init__(id, *args, **kwargs)
+        self._client = ovh.Client(
+            endpoint=endpoint,
+            application_key=application_key,
+            application_secret=application_secret,
+            consumer_key=consumer_key,
+        )
+
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+        zone_name = zone.name[:-1]
+        records = self.get_records(zone_name=zone_name)
+
+        values = defaultdict(lambda: defaultdict(list))
+        for record in records:
+            values[record['subDomain']][record['fieldType']].append(record)
+
+        before = len(zone.records)
+        for name, types in values.items():
+            for _type, records in types.items():
+                data_for = getattr(self, '_data_for_{}'.format(_type))
+                record = Record.new(zone, name, data_for(_type, records),
+                                    source=self, lenient=lenient)
+                zone.add_record(record)
+
+        self.log.info('populate:   found %s records',
+                      len(zone.records) - before)
+
+    def _apply(self, plan):
+        desired = plan.desired
+        changes = plan.changes
+        zone_name = desired.name[:-1]
+        self.log.info('_apply: zone=%s, len(changes)=%d', desired.name,
+                      len(changes))
+        for change in changes:
+            class_name = change.__class__.__name__
+            getattr(self, '_apply_{}'.format(class_name).lower())(zone_name,
+                                                                  change)
+
+        # We need to refresh the zone to really apply the changes
+        self._client.post('/domain/zone/{}/refresh'.format(zone_name))
+
+    def _apply_create(self, zone_name, change):
+        new = change.new
+        params_for = getattr(self, '_params_for_{}'.format(new._type))
+        for params in params_for(new):
+            self.create_record(zone_name, params)
+
+    def _apply_update(self, zone_name, change):
+        self._apply_delete(zone_name, change)
+        self._apply_create(zone_name, change)
+
+    def _apply_delete(self, zone_name, change):
+        existing = change.existing
+        self.delete_records(zone_name, existing._type, existing.name)
+
+    @staticmethod
+    def _data_for_multiple(_type, records):
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': [record['target'] for record in records]
+        }
+
+    @staticmethod
+    def _data_for_single(_type, records):
+        record = records[0]
+        return {
+            'ttl': record['ttl'],
+            'type': _type,
+            'value': record['target']
+        }
+
+    @staticmethod
+    def _data_for_MX(_type, records):
+        values = []
+        for record in records:
+            preference, exchange = record['target'].split(' ', 1)
+            values.append({
+                'preference': preference,
+                'exchange': exchange,
+            })
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values,
+        }
+
+    @staticmethod
+    def _data_for_NAPTR(_type, records):
+        values = []
+        for record in records:
+            order, preference, flags, service, regexp, replacement = record[
+                'target'].split(' ', 5)
+            values.append({
+                'flags': flags[1:-1],
+                'order': order,
+                'preference': preference,
+                'regexp': regexp[1:-1],
+                'replacement': replacement,
+                'service': service[1:-1],
+            })
+        return {
+            'type': _type,
+            'ttl': records[0]['ttl'],
+            'values': values
+        }
+
+    @staticmethod
+    def _data_for_SRV(_type, records):
+        values = []
+        for record in records:
+            priority, weight, port, target = record['target'].split(' ', 3)
+            values.append({
+                'port': port,
+                'priority': priority,
+                'target': '{}.'.format(target),
+                'weight': weight
+            })
+        return {
+            'type': _type,
+            'ttl': records[0]['ttl'],
+            'values': values
+        }
+
+    @staticmethod
+    def _data_for_SSHFP(_type, records):
+        values = []
+        for record in records:
+            algorithm, fingerprint_type, fingerprint = record['target'].split(
+                ' ', 2)
+            values.append({
+                'algorithm': algorithm,
+                'fingerprint': fingerprint,
+                'fingerprint_type': fingerprint_type
+            })
+        return {
+            'type': _type,
+            'ttl': records[0]['ttl'],
+            'values': values
+        }
+
+    _data_for_A = _data_for_multiple
+    _data_for_AAAA = _data_for_multiple
+    _data_for_NS = _data_for_multiple
+    _data_for_TXT = _data_for_multiple
+    _data_for_SPF = _data_for_multiple
+    _data_for_PTR = _data_for_single
+    _data_for_CNAME = _data_for_single
+
+    @staticmethod
+    def _params_for_multiple(record):
+        for value in record.values:
+            yield {
+                'target': value,
+                'subDomain': record.name,
+                'ttl': record.ttl,
+                'fieldType': record._type,
+            }
+
+    @staticmethod
+    def _params_for_single(record):
+        yield {
+            'target': record.value,
+            'subDomain': record.name,
+            'ttl': record.ttl,
+            'fieldType': record._type
+        }
+
+    @staticmethod
+    def _params_for_MX(record):
+        for value in record.values:
+            yield {
+                'target': '%d %s' % (value.preference, value.exchange),
+                'subDomain': record.name,
+                'ttl': record.ttl,
+                'fieldType': record._type
+            }
+
+    @staticmethod
+    def _params_for_NAPTR(record):
+        for value in record.values:
+            content = '{} {} "{}" "{}" "{}" {}' \
+                .format(value.order, value.preference, value.flags,
+                        value.service, value.regexp, value.replacement)
+            yield {
+                'target': content,
+                'subDomain': record.name,
+                'ttl': record.ttl,
+                'fieldType': record._type
+            }
+
+    @staticmethod
+    def _params_for_SRV(record):
+        for value in record.values:
+            yield {
+                'subDomain': '{} {} {} {}'.format(value.priority,
+                                                  value.weight, value.port,
+                                                  value.target),
+                'target': record.name,
+                'ttl': record.ttl,
+                'fieldType': record._type
+            }
+
+    @staticmethod
+    def _params_for_SSHFP(record):
+        for value in record.values:
+            yield {
+                'subDomain': '{} {} {}'.format(value.algorithm,
+                                               value.fingerprint_type,
+                                               value.fingerprint),
+                'target': record.name,
+                'ttl': record.ttl,
+                'fieldType': record._type
+            }
+
+    _params_for_A = _params_for_multiple
+    _params_for_AAAA = _params_for_multiple
+    _params_for_NS = _params_for_multiple
+    _params_for_SPF = _params_for_multiple
+    _params_for_TXT = _params_for_multiple
+
+    _params_for_CNAME = _params_for_single
+    _params_for_PTR = _params_for_single
+
+    def get_records(self, zone_name):
+        """
+        List all records of a DNS zone
+        :param zone_name: Name of zone
+        :return: list of id's records
+        """
+        records = self._client.get('/domain/zone/{}/record'.format(zone_name))
+        return [self.get_record(zone_name, record_id) for record_id in records]
+
+    def get_record(self, zone_name, record_id):
+        """
+        Get record with given id
+        :param zone_name: Name of the zone
+        :param record_id: Id of the record
+        :return: Value of the record
+        """
+        return self._client.get(
+            '/domain/zone/{}/record/{}'.format(zone_name, record_id))
+
+    def delete_records(self, zone_name, record_type, subdomain):
+        """
+        Delete record from have fieldType=type and subDomain=subdomain
+        :param zone_name: Name of the zone
+        :param record_type: fieldType
+        :param subdomain: subDomain
+        """
+        records = self._client.get('/domain/zone/{}/record'.format(zone_name),
+                                   fieldType=record_type, subDomain=subdomain)
+        for record in records:
+            self.delete_record(zone_name, record)
+
+    def delete_record(self, zone_name, record_id):
+        """
+        Delete record with a given id
+        :param zone_name: Name of the zone
+        :param record_id: Id of the record
+        """
+        self.log.debug('Delete record: zone: %s, id %s', zone_name,
+                       record_id)
+        self._client.delete(
+            '/domain/zone/{}/record/{}'.format(zone_name, record_id))
+
+    def create_record(self, zone_name, params):
+        """
+        Create a record
+        :param zone_name: Name of the zone
+        :param params: {'fieldType': 'A', 'ttl': 60, 'subDomain': 'www',
+        'target': '1.2.3.4'
+        """
+        self.log.debug('Create record: zone: %s, id %s', zone_name,
+                       params)
+        return self._client.post('/domain/zone/{}/record'.format(zone_name),
+                                 **params)

octodns/provider/powerdns.py

@@ -14,13 +14,17 @@ from .base import BaseProvider
 
 class PowerDnsBaseProvider(BaseProvider):
     SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS',
+                    'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT'))
     TIMEOUT = 5
 
-    def __init__(self, id, host, api_key, port=8081, *args, **kwargs):
+    def __init__(self, id, host, api_key, port=8081, scheme="http", *args,
+                 **kwargs):
         super(PowerDnsBaseProvider, self).__init__(id, *args, **kwargs)
 
         self.host = host
         self.port = port
+        self.scheme = scheme
 
         sess = Session()
         sess.headers.update({'X-API-Key': api_key})
@@ -29,8 +33,8 @@ class PowerDnsBaseProvider(BaseProvider):
     def _request(self, method, path, data=None):
         self.log.debug('_request: method=%s, path=%s', method, path)
 
-        url = 'http://{}:{}/api/v1/servers/localhost/{}' \
-            .format(self.host, self.port, path)
+        url = '{}://{}:{}/api/v1/servers/localhost/{}' \
+            .format(self.scheme, self.host, self.port, path)
         resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT)
         self.log.debug('_request:   status=%d', resp.status_code)
         resp.raise_for_status()
@@ -57,6 +61,21 @@ class PowerDnsBaseProvider(BaseProvider):
     _data_for_AAAA = _data_for_multiple
     _data_for_NS = _data_for_multiple
 
+    def _data_for_CAA(self, rrset):
+        values = []
+        for record in rrset['records']:
+            flags, tag, value = record['content'].split(' ', 2)
+            values.append({
+                'flags': flags,
+                'tag': tag,
+                'value': value[1:-1],
+            })
+        return {
+            'type': rrset['type'],
+            'values': values,
+            'ttl': rrset['ttl']
+        }
+
     def _data_for_single(self, rrset):
         return {
             'type': rrset['type'],
@@ -81,10 +100,10 @@ class PowerDnsBaseProvider(BaseProvider):
     def _data_for_MX(self, rrset):
         values = []
         for record in rrset['records']:
-            priority, value = record['content'].split(' ', 1)
+            preference, exchange = record['content'].split(' ', 1)
             values.append({
-                'priority': priority,
-                'value': value,
+                'preference': preference,
+                'exchange': exchange,
             })
         return {
             'type': rrset['type'],
@@ -144,8 +163,9 @@ class PowerDnsBaseProvider(BaseProvider):
             'ttl': rrset['ttl']
         }
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: name=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
 
         resp = None
         try:
@@ -175,7 +195,7 @@ class PowerDnsBaseProvider(BaseProvider):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 record_name = zone.hostname_from_fqdn(rrset['name'])
                 record = Record.new(zone, record_name, data_for(rrset),
-                                    source=self)
+                                    source=self, lenient=lenient)
                 zone.add_record(record)
 
         self.log.info('populate:   found %s records',
@@ -189,6 +209,12 @@ class PowerDnsBaseProvider(BaseProvider):
     _records_for_AAAA = _records_for_multiple
     _records_for_NS = _records_for_multiple
 
+    def _records_for_CAA(self, record):
+        return [{
+            'content': '{} {} "{}"'.format(v.flags, v.tag, v.value),
+            'disabled': False
+        } for v in record.values]
+
     def _records_for_single(self, record):
         return [{'content': record.value, 'disabled': False}]
 
@@ -205,7 +231,7 @@ class PowerDnsBaseProvider(BaseProvider):
 
     def _records_for_MX(self, record):
         return [{
-            'content': '{} {}'.format(v.priority, v.value),
+            'content': '{} {}'.format(v.preference, v.exchange),
             'disabled': False
         } for v in record.values]
 

octodns/provider/route53.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
 from boto3 import client
+from botocore.config import Config
 from collections import defaultdict
 from incf.countryutils.transformations import cca_to_ctca2
 from uuid import uuid4
@@ -89,6 +90,10 @@ class _Route53Record(object):
     _values_for_AAAA = _values_for_values
     _values_for_NS = _values_for_values
 
+    def _values_for_CAA(self, record):
+        return ['{} {} "{}"'.format(v.flags, v.tag, v.value)
+                for v in record.values]
+
     def _values_for_value(self, record):
         return [record.value]
 
@@ -96,7 +101,8 @@ class _Route53Record(object):
     _values_for_PTR = _values_for_value
 
     def _values_for_MX(self, record):
-        return ['{} {}'.format(v.priority, v.value) for v in record.values]
+        return ['{} {}'.format(v.preference, v.exchange)
+                for v in record.values]
 
     def _values_for_NAPTR(self, record):
         return ['{} {} "{}" "{}" "{}" {}'
@@ -108,8 +114,7 @@ class _Route53Record(object):
                 for v in record.values]
 
     def _values_for_quoted(self, record):
-        return ['"{}"'.format(v.replace('"', '\\"'))
-                for v in record.values]
+        return record.chunked_values
 
     _values_for_SPF = _values_for_quoted
     _values_for_TXT = _values_for_quoted
@@ -220,28 +225,35 @@ class Route53Provider(BaseProvider):
     In general the account used will need full permissions on Route53.
     '''
     SUPPORTS_GEO = True
+    SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR',
+                    'SPF', 'SRV', 'TXT'))
 
     # This should be bumped when there are underlying changes made to the
     # health check config.
     HEALTH_CHECK_VERSION = '0001'
 
     def __init__(self, id, access_key_id, secret_access_key, max_changes=1000,
-                 *args, **kwargs):
+                 client_max_attempts=None, *args, **kwargs):
         self.max_changes = max_changes
         self.log = logging.getLogger('Route53Provider[{}]'.format(id))
         self.log.debug('__init__: id=%s, access_key_id=%s, '
                        'secret_access_key=***', id, access_key_id)
         super(Route53Provider, self).__init__(id, *args, **kwargs)
+
+        config = None
+        if client_max_attempts is not None:
+            self.log.info('__init__: setting max_attempts to %d',
+                          client_max_attempts)
+            config = Config(retries={'max_attempts': client_max_attempts})
+
         self._conn = client('route53', aws_access_key_id=access_key_id,
-                            aws_secret_access_key=secret_access_key)
+                            aws_secret_access_key=secret_access_key,
+                            config=config)
 
         self._r53_zones = None
         self._r53_rrsets = {}
         self._health_checks = None
 
-    def supports(self, record):
-        return record._type not in ('ALIAS', 'SSHFP')
-
     @property
     def r53_zones(self):
         if self._r53_zones is None:
@@ -250,7 +262,7 @@ class Route53Provider(BaseProvider):
             more = True
             start = {}
             while more:
-                resp = self._conn.list_hosted_zones()
+                resp = self._conn.list_hosted_zones(**start)
                 for z in resp['HostedZones']:
                     zones[z['Name']] = z['Id']
                 more = resp['IsTruncated']
@@ -310,6 +322,21 @@ class Route53Provider(BaseProvider):
     _data_for_A = _data_for_geo
     _data_for_AAAA = _data_for_geo
 
+    def _data_for_CAA(self, rrset):
+        values = []
+        for rr in rrset['ResourceRecords']:
+            flags, tag, value = rr['Value'].split(' ')
+            values.append({
+                'flags': flags,
+                'tag': tag,
+                'value': value[1:-1],
+            })
+        return {
+            'type': rrset['Type'],
+            'values': values,
+            'ttl': int(rrset['TTL'])
+        }
+
     def _data_for_single(self, rrset):
         return {
             'type': rrset['Type'],
@@ -336,10 +363,10 @@ class Route53Provider(BaseProvider):
     def _data_for_MX(self, rrset):
         values = []
         for rr in rrset['ResourceRecords']:
-            priority, value = rr['Value'].split(' ')
+            preference, exchange = rr['Value'].split(' ')
             values.append({
-                'priority': priority,
-                'value': value,
+                'preference': preference,
+                'exchange': exchange,
             })
         return {
             'type': rrset['Type'],
@@ -419,8 +446,10 @@ class Route53Provider(BaseProvider):
 
         return self._r53_rrsets[zone_id]
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: name=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+
         before = len(zone.records)
 
         zone_id = self._get_zone_id(zone.name)
@@ -450,7 +479,8 @@ class Route53Provider(BaseProvider):
                         data['geo'] = geo
                     else:
                         data = data[0]
-                    record = Record.new(zone, name, data, source=self)
+                    record = Record.new(zone, name, data, source=self,
+                                        lenient=lenient)
                     zone.add_record(record)
 
         self.log.info('populate:   found %s records',

octodns/provider/yaml.py

@@ -31,6 +31,8 @@ class YamlProvider(BaseProvider):
         enforce_order: True
     '''
     SUPPORTS_GEO = True
+    SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR',
+                   'SSHFP', 'SPF', 'SRV', 'TXT'))
 
     def __init__(self, id, directory, default_ttl=3600, enforce_order=True,
                  *args, **kwargs):
@@ -43,8 +45,10 @@ class YamlProvider(BaseProvider):
         self.default_ttl = default_ttl
         self.enforce_order = enforce_order
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: zone=%s, target=%s', zone.name, target)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+
         if target:
             # When acting as a target we ignore any existing records so that we
             # create a completely new copy
@@ -61,7 +65,8 @@ class YamlProvider(BaseProvider):
                     for d in data:
                         if 'ttl' not in d:
                             d['ttl'] = self.default_ttl
-                        record = Record.new(zone, name, d, source=self)
+                        record = Record.new(zone, name, d, source=self,
+                                            lenient=lenient)
                         zone.add_record(record)
 
         self.log.info('populate:   found %s records',

octodns/record.py

@@ -54,51 +54,64 @@ class Delete(Change):
         return 'Delete {}'.format(self.existing)
 
 
-_unescaped_semicolon_re = re.compile(r'\w;')
+class ValidationError(Exception):
+
+    @classmethod
+    def build_message(cls, fqdn, reasons):
+        return 'Invalid record {}\n  - {}'.format(fqdn, '\n  - '.join(reasons))
+
+    def __init__(self, fqdn, reasons):
+        super(Exception, self).__init__(self.build_message(fqdn, reasons))
+        self.fqdn = fqdn
+        self.reasons = reasons
 
 
 class Record(object):
     log = getLogger('Record')
 
     @classmethod
-    def new(cls, zone, name, data, source=None):
+    def new(cls, zone, name, data, source=None, lenient=False):
+        fqdn = '{}.{}'.format(name, zone.name) if name else zone.name
         try:
             _type = data['type']
         except KeyError:
-            fqdn = '{}.{}'.format(name, zone.name) if name else zone.name
             raise Exception('Invalid record {}, missing type'.format(fqdn))
         try:
-            _type = {
+            _class = {
                 'A': ARecord,
                 'AAAA': AaaaRecord,
                 'ALIAS': AliasRecord,
-                # cert
+                'CAA': CaaRecord,
                 'CNAME': CnameRecord,
-                # dhcid
-                # dname
-                # dnskey
-                # ds
-                # ipseckey
-                # key
-                # kx
-                # loc
                 'MX': MxRecord,
                 'NAPTR': NaptrRecord,
                 'NS': NsRecord,
-                # nsap
                 'PTR': PtrRecord,
-                # px
-                # rp
-                # soa - would it even make sense?
                 'SPF': SpfRecord,
                 'SRV': SrvRecord,
                 'SSHFP': SshfpRecord,
                 'TXT': TxtRecord,
-                # url
             }[_type]
         except KeyError:
             raise Exception('Unknown record type: "{}"'.format(_type))
-        return _type(zone, name, data, source=source)
+        reasons = _class.validate(name, data)
+        if reasons:
+            if lenient:
+                cls.log.warn(ValidationError.build_message(fqdn, reasons))
+            else:
+                raise ValidationError(fqdn, reasons)
+        return _class(zone, name, data, source=source)
+
+    @classmethod
+    def validate(cls, name, data):
+        reasons = []
+        try:
+            ttl = int(data['ttl'])
+            if ttl < 0:
+                reasons.append('invalid ttl')
+        except KeyError:
+            reasons.append('missing ttl')
+        return reasons
 
     def __init__(self, zone, name, data, source=None):
         self.log.debug('__init__: zone.name=%s, type=%11s, name=%s', zone.name,
@@ -106,11 +119,8 @@ class Record(object):
         self.zone = zone
         # force everything lower-case just to be safe
         self.name = str(name).lower() if name else name
-        try:
-            self.ttl = int(data['ttl'])
-        except KeyError:
-            raise Exception('Invalid record {}, missing ttl'.format(self.fqdn))
         self.source = source
+        self.ttl = int(data['ttl'])
 
         self._octodns = data.get('octodns', {})
 
@@ -171,11 +181,17 @@ class GeoValue(object):
     geo_re = re.compile(r'^(?P<continent_code>\w\w)(-(?P<country_code>\w\w)'
                         r'(-(?P<subdivision_code>\w\w))?)?$')
 
-    def __init__(self, geo, values):
-        match = self.geo_re.match(geo)
+    @classmethod
+    def _validate_geo(cls, code):
+        reasons = []
+        match = cls.geo_re.match(code)
         if not match:
-            raise Exception('Invalid geo "{}"'.format(geo))
+            reasons.append('invalid geo "{}"'.format(code))
+        return reasons
+
+    def __init__(self, geo, values):
         self.code = geo
+        match = self.geo_re.match(geo)
         self.continent_code = match.group('continent_code')
         self.country_code = match.group('country_code')
         self.subdivision_code = match.group('subdivision_code')
@@ -202,16 +218,29 @@ class GeoValue(object):
 
 class _ValuesMixin(object):
 
-    def __init__(self, zone, name, data, source=None):
-        super(_ValuesMixin, self).__init__(zone, name, data, source=source)
+    @classmethod
+    def validate(cls, name, data):
+        reasons = super(_ValuesMixin, cls).validate(name, data)
+        values = []
         try:
             values = data['values']
         except KeyError:
             try:
                 values = [data['value']]
             except KeyError:
-                raise Exception('Invalid record {}, missing value(s)'
-                                .format(self.fqdn))
+                reasons.append('missing value(s)')
+
+        for value in values:
+            reasons.extend(cls._validate_value(value))
+
+        return reasons
+
+    def __init__(self, zone, name, data, source=None):
+        super(_ValuesMixin, self).__init__(zone, name, data, source=source)
+        try:
+            values = data['values']
+        except KeyError:
+            values = [data['value']]
         self.values = sorted(self._process_values(values))
 
     def changes(self, other, target):
@@ -229,9 +258,10 @@ class _ValuesMixin(object):
         return ret
 
     def __repr__(self):
+        values = "['{}']".format("', '".join([str(v) for v in self.values]))
         return '<{} {} {}, {}, {}>'.format(self.__class__.__name__,
                                            self._type, self.ttl,
-                                           self.fqdn, self.values)
+                                           self.fqdn, values)
 
 
 class _GeoMixin(_ValuesMixin):
@@ -241,6 +271,21 @@ class _GeoMixin(_ValuesMixin):
     Must be included before `Record`.
     '''
 
+    @classmethod
+    def validate(cls, name, data):
+        reasons = super(_GeoMixin, cls).validate(name, data)
+        try:
+            geo = dict(data['geo'])
+            # TODO: validate legal codes
+            for code, values in geo.items():
+                reasons.extend(GeoValue._validate_geo(code))
+                for value in values:
+                    reasons.extend(cls._validate_value(value))
+        except KeyError:
+            pass
+        return reasons
+
+    # TODO: support 'value' as well
     # TODO: move away from "data" hash to strict params, it's kind of leaking
     # the yaml implementation into here and then forcing it back out into
     # non-yaml providers during input
@@ -250,9 +295,8 @@ class _GeoMixin(_ValuesMixin):
             self.geo = dict(data['geo'])
         except KeyError:
             self.geo = {}
-        for k, vs in self.geo.items():
-            vs = sorted(self._process_values(vs))
-            self.geo[k] = GeoValue(k, vs)
+        for code, values in self.geo.items():
+            self.geo[code] = GeoValue(code, values)
 
     def _data(self):
         ret = super(_GeoMixin, self)._data()
@@ -281,41 +325,52 @@ class _GeoMixin(_ValuesMixin):
 class ARecord(_GeoMixin, Record):
     _type = 'A'
 
+    @classmethod
+    def _validate_value(self, value):
+        reasons = []
+        try:
+            IPv4Address(unicode(value))
+        except Exception:
+            reasons.append('invalid ip address "{}"'.format(value))
+        return reasons
+
     def _process_values(self, values):
-        for ip in values:
-            try:
-                IPv4Address(unicode(ip))
-            except Exception:
-                raise Exception('Invalid record {}, value {} not a valid ip'
-                                .format(self.fqdn, ip))
         return values
 
 
 class AaaaRecord(_GeoMixin, Record):
     _type = 'AAAA'
 
+    @classmethod
+    def _validate_value(self, value):
+        reasons = []
+        try:
+            IPv6Address(unicode(value))
+        except Exception:
+            reasons.append('invalid ip address "{}"'.format(value))
+        return reasons
+
     def _process_values(self, values):
-        ret = []
-        for ip in values:
-            try:
-                IPv6Address(unicode(ip))
-                ret.append(ip.lower())
-            except Exception:
-                raise Exception('Invalid record {}, value {} not a valid ip'
-                                .format(self.fqdn, ip))
-        return ret
+        return values
 
 
 class _ValueMixin(object):
 
-    def __init__(self, zone, name, data, source=None):
-        super(_ValueMixin, self).__init__(zone, name, data, source=source)
+    @classmethod
+    def validate(cls, name, data):
+        reasons = super(_ValueMixin, cls).validate(name, data)
+        value = None
         try:
             value = data['value']
         except KeyError:
-            raise Exception('Invalid record {}, missing value'
-                            .format(self.fqdn))
-        self.value = self._process_value(value)
+            reasons.append('missing value')
+        if value:
+            reasons.extend(cls._validate_value(value))
+        return reasons
+
+    def __init__(self, zone, name, data, source=None):
+        super(_ValueMixin, self).__init__(zone, name, data, source=source)
+        self.value = self._process_value(data['value'])
 
     def changes(self, other, target):
         if self.value != other.value:
@@ -336,62 +391,187 @@ class _ValueMixin(object):
 class AliasRecord(_ValueMixin, Record):
     _type = 'ALIAS'
 
-    def _process_value(self, value):
+    @classmethod
+    def _validate_value(self, value):
+        reasons = []
         if not value.endswith('.'):
-            raise Exception('Invalid record {}, value ({}) missing trailing .'
-                            .format(self.fqdn, value))
+            reasons.append('missing trailing .')
+        return reasons
+
+    def _process_value(self, value):
         return value
 
 
+class CaaValue(object):
+    # https://tools.ietf.org/html/rfc6844#page-5
+
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
+        try:
+            flags = int(value.get('flags', 0))
+            if flags < 0 or flags > 255:
+                reasons.append('invalid flags "{}"'.format(flags))
+        except ValueError:
+            reasons.append('invalid flags "{}"'.format(value['flags']))
+
+        if 'tag' not in value:
+            reasons.append('missing tag')
+        if 'value' not in value:
+            reasons.append('missing value')
+
+        return reasons
+
+    def __init__(self, value):
+        self.flags = int(value.get('flags', 0))
+        self.tag = value['tag']
+        self.value = value['value']
+
+    @property
+    def data(self):
+        return {
+            'flags': self.flags,
+            'tag': self.tag,
+            'value': self.value,
+        }
+
+    def __cmp__(self, other):
+        if self.flags == other.flags:
+            if self.tag == other.tag:
+                return cmp(self.value, other.value)
+            return cmp(self.tag, other.tag)
+        return cmp(self.flags, other.flags)
+
+    def __repr__(self):
+        return '{} {} "{}"'.format(self.flags, self.tag, self.value)
+
+
+class CaaRecord(_ValuesMixin, Record):
+    _type = 'CAA'
+
+    @classmethod
+    def _validate_value(cls, value):
+        return CaaValue._validate_value(value)
+
+    def _process_values(self, values):
+        return [CaaValue(v) for v in values]
+
+
 class CnameRecord(_ValueMixin, Record):
     _type = 'CNAME'
 
-    def _process_value(self, value):
+    @classmethod
+    def validate(cls, name, data):
+        reasons = []
+        if name == '':
+            reasons.append('root CNAME not allowed')
+        reasons.extend(super(CnameRecord, cls).validate(name, data))
+        return reasons
+
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
         if not value.endswith('.'):
-            raise Exception('Invalid record {}, value ({}) missing trailing .'
-                            .format(self.fqdn, value))
-        return value.lower()
+            reasons.append('missing trailing .')
+        return reasons
+
+    def _process_value(self, value):
+        return value
 
 
 class MxValue(object):
 
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
+        try:
+            int(value.get('preference', None) or value['priority'])
+        except KeyError:
+            reasons.append('missing preference')
+        except ValueError:
+            reasons.append('invalid preference "{}"'
+                           .format(value['preference']))
+        exchange = None
+        try:
+            exchange = value.get('exchange', None) or value['value']
+            if not exchange.endswith('.'):
+                reasons.append('missing trailing .')
+        except KeyError:
+            reasons.append('missing exchange')
+        return reasons
+
     def __init__(self, value):
-        # TODO: rename preference
-        self.priority = int(value['priority'])
-        # TODO: rename to exchange?
-        self.value = value['value'].lower()
+        # RFC1035 says preference, half the providers use priority
+        try:
+            preference = value['preference']
+        except KeyError:
+            preference = value['priority']
+        self.preference = int(preference)
+        # UNTIL 1.0 remove value fallback
+        try:
+            exchange = value['exchange']
+        except KeyError:
+            exchange = value['value']
+        self.exchange = exchange
 
     @property
     def data(self):
         return {
-            'priority': self.priority,
-            'value': self.value,
+            'preference': self.preference,
+            'exchange': self.exchange,
         }
 
     def __cmp__(self, other):
-        if self.priority == other.priority:
-            return cmp(self.value, other.value)
-        return cmp(self.priority, other.priority)
+        if self.preference == other.preference:
+            return cmp(self.exchange, other.exchange)
+        return cmp(self.preference, other.preference)
 
     def __repr__(self):
-        return "'{} {}'".format(self.priority, self.value)
+        return "'{} {}'".format(self.preference, self.exchange)
 
 
 class MxRecord(_ValuesMixin, Record):
     _type = 'MX'
 
+    @classmethod
+    def _validate_value(cls, value):
+        return MxValue._validate_value(value)
+
     def _process_values(self, values):
-        ret = []
-        for value in values:
-            try:
-                ret.append(MxValue(value))
-            except KeyError as e:
-                raise Exception('Invalid value in record {}, missing {}'
-                                .format(self.fqdn, e.args[0]))
-        return ret
+        return [MxValue(v) for v in values]
 
 
 class NaptrValue(object):
+    VALID_FLAGS = ('S', 'A', 'U', 'P')
+
+    @classmethod
+    def _validate_value(cls, data):
+        reasons = []
+        try:
+            int(data['order'])
+        except KeyError:
+            reasons.append('missing order')
+        except ValueError:
+            reasons.append('invalid order "{}"'.format(data['order']))
+        try:
+            int(data['preference'])
+        except KeyError:
+            reasons.append('missing preference')
+        except ValueError:
+            reasons.append('invalid preference "{}"'
+                           .format(data['preference']))
+        try:
+            flags = data['flags']
+            if flags not in cls.VALID_FLAGS:
+                reasons.append('unrecognized flags "{}"'.format(flags))
+        except KeyError:
+            reasons.append('missing flags')
+
+        # TODO: validate these... they're non-trivial
+        for k in ('service', 'regexp', 'replacement'):
+            if k not in data:
+                reasons.append('missing {}'.format(k))
+        return reasons
 
     def __init__(self, value):
         self.order = int(value['order'])
@@ -437,41 +617,70 @@ class NaptrValue(object):
 class NaptrRecord(_ValuesMixin, Record):
     _type = 'NAPTR'
 
+    @classmethod
+    def _validate_value(cls, value):
+        return NaptrValue._validate_value(value)
+
     def _process_values(self, values):
-        ret = []
-        for value in values:
-            try:
-                ret.append(NaptrValue(value))
-            except KeyError as e:
-                raise Exception('Invalid value in record {}, missing {}'
-                                .format(self.fqdn, e.args[0]))
-        return ret
+        return [NaptrValue(v) for v in values]
 
 
 class NsRecord(_ValuesMixin, Record):
     _type = 'NS'
 
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
+        if not value.endswith('.'):
+            reasons.append('missing trailing .')
+        return reasons
+
     def _process_values(self, values):
-        ret = []
-        for ns in values:
-            if not ns.endswith('.'):
-                raise Exception('Invalid record {}, value {} missing '
-                                'trailing .'.format(self.fqdn, ns))
-            ret.append(ns.lower())
-        return ret
+        return values
 
 
 class PtrRecord(_ValueMixin, Record):
     _type = 'PTR'
 
-    def _process_value(self, value):
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
         if not value.endswith('.'):
-            raise Exception('Invalid record {}, value ({}) missing trailing .'
-                            .format(self.fqdn, value))
-        return value.lower()
+            reasons.append('missing trailing .')
+        return reasons
+
+    def _process_value(self, value):
+        return value
 
 
 class SshfpValue(object):
+    VALID_ALGORITHMS = (1, 2)
+    VALID_FINGERPRINT_TYPES = (1,)
+
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
+        try:
+            algorithm = int(value['algorithm'])
+            if algorithm not in cls.VALID_ALGORITHMS:
+                reasons.append('unrecognized algorithm "{}"'.format(algorithm))
+        except KeyError:
+            reasons.append('missing algorithm')
+        except ValueError:
+            reasons.append('invalid algorithm "{}"'.format(value['algorithm']))
+        try:
+            fingerprint_type = int(value['fingerprint_type'])
+            if fingerprint_type not in cls.VALID_FINGERPRINT_TYPES:
+                reasons.append('unrecognized fingerprint_type "{}"'
+                               .format(fingerprint_type))
+        except KeyError:
+            reasons.append('missing fingerprint_type')
+        except ValueError:
+            reasons.append('invalid fingerprint_type "{}"'
+                           .format(value['fingerprint_type']))
+        if 'fingerprint' not in value:
+            reasons.append('missing fingerprint')
+        return reasons
 
     def __init__(self, value):
         self.algorithm = int(value['algorithm'])
@@ -501,26 +710,81 @@ class SshfpValue(object):
 class SshfpRecord(_ValuesMixin, Record):
     _type = 'SSHFP'
 
+    @classmethod
+    def _validate_value(cls, value):
+        return SshfpValue._validate_value(value)
+
+    def _process_values(self, values):
+        return [SshfpValue(v) for v in values]
+
+
+_unescaped_semicolon_re = re.compile(r'\w;')
+
+
+class _ChunkedValuesMixin(_ValuesMixin):
+    CHUNK_SIZE = 255
+
+    @classmethod
+    def _validate_value(cls, value):
+        if _unescaped_semicolon_re.search(value):
+            return ['unescaped ;']
+        return []
+
     def _process_values(self, values):
         ret = []
-        for value in values:
-            try:
-                ret.append(SshfpValue(value))
-            except KeyError as e:
-                raise Exception('Invalid value in record {}, missing {}'
-                                .format(self.fqdn, e.args[0]))
+        for v in values:
+            if v and v[0] == '"':
+                v = v[1:-1]
+            ret.append(v.replace('" "', ''))
         return ret
 
-
-class SpfRecord(_ValuesMixin, Record):
-    _type = 'SPF'
-
-    def _process_values(self, values):
+    @property
+    def chunked_values(self):
+        values = []
+        for v in self.values:
+            v = v.replace('"', '\\"')
+            vs = [v[i:i + self.CHUNK_SIZE]
+                  for i in range(0, len(v), self.CHUNK_SIZE)]
+            vs = '" "'.join(vs)
+            values.append('"{}"'.format(vs))
         return values
 
 
+class SpfRecord(_ChunkedValuesMixin, Record):
+    _type = 'SPF'
+
+
 class SrvValue(object):
 
+    @classmethod
+    def _validate_value(self, value):
+        reasons = []
+        # TODO: validate algorithm and fingerprint_type values
+        try:
+            int(value['priority'])
+        except KeyError:
+            reasons.append('missing priority')
+        except ValueError:
+            reasons.append('invalid priority "{}"'.format(value['priority']))
+        try:
+            int(value['weight'])
+        except KeyError:
+            reasons.append('missing weight')
+        except ValueError:
+            reasons.append('invalid weight "{}"'.format(value['weight']))
+        try:
+            int(value['port'])
+        except KeyError:
+            reasons.append('missing port')
+        except ValueError:
+            reasons.append('invalid port "{}"'.format(value['port']))
+        try:
+            if not value['target'].endswith('.'):
+                reasons.append('missing trailing .')
+        except KeyError:
+            reasons.append('missing target')
+        return reasons
+
     def __init__(self, value):
         self.priority = int(value['priority'])
         self.weight = int(value['weight'])
@@ -554,28 +818,21 @@ class SrvRecord(_ValuesMixin, Record):
     _type = 'SRV'
     _name_re = re.compile(r'^_[^\.]+\.[^\.]+')
 
-    def __init__(self, zone, name, data, source=None):
-        if not self._name_re.match(name):
-            raise Exception('Invalid name {}.{}'.format(name, zone.name))
-        super(SrvRecord, self).__init__(zone, name, data, source)
+    @classmethod
+    def validate(cls, name, data):
+        reasons = []
+        if not cls._name_re.match(name):
+            reasons.append('invalid name')
+        reasons.extend(super(SrvRecord, cls).validate(name, data))
+        return reasons
+
+    @classmethod
+    def _validate_value(cls, value):
+        return SrvValue._validate_value(value)
 
     def _process_values(self, values):
-        ret = []
-        for value in values:
-            try:
-                ret.append(SrvValue(value))
-            except KeyError as e:
-                raise Exception('Invalid value in record {}, missing {}'
-                                .format(self.fqdn, e.args[0]))
-        return ret
+        return [SrvValue(v) for v in values]
 
 
-class TxtRecord(_ValuesMixin, Record):
+class TxtRecord(_ChunkedValuesMixin, Record):
     _type = 'TXT'
-
-    def _process_values(self, values):
-        for value in values:
-            if _unescaped_semicolon_re.search(value):
-                raise Exception('Invalid record {}, unescaped ;'
-                                .format(self.fqdn))
-        return values

octodns/source/base.py

@@ -16,18 +16,27 @@ class BaseSource(object):
         if not hasattr(self, 'SUPPORTS_GEO'):
             raise NotImplementedError('Abstract base class, SUPPORTS_GEO '
                                       'property missing')
+        if not hasattr(self, 'SUPPORTS'):
+            raise NotImplementedError('Abstract base class, SUPPORTS '
+                                      'property missing')
 
-    def populate(self, zone, target=False):
+    def populate(self, zone, target=False, lenient=False):
         '''
         Loads all zones the provider knows about
+
+        When `target` is True the populate call is being made to load the
+        current state of the provider.
+
+        When `lenient` is True the populate call may skip record validation and
+        do a "best effort" load of data. That will allow through some common,
+        but not best practices stuff that we otherwise would reject. E.g. no
+        trailing . or mising escapes for ;.
         '''
         raise NotImplementedError('Abstract base class, populate method '
                                   'missing')
 
     def supports(self, record):
-        # Unless overriden and handled appropriaitely we'll assume that all
-        # record types are supported
-        return True
+        return record._type in self.SUPPORTS
 
     def __repr__(self):
         return self.__class__.__name__

octodns/source/tinydns.py

@@ -19,6 +19,7 @@ from .base import BaseSource
 
 class TinyDnsBaseSource(BaseSource):
     SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'CNAME', 'MX', 'NS'))
 
     split_re = re.compile(r':+')
 
@@ -64,8 +65,8 @@ class TinyDnsBaseSource(BaseSource):
             'ttl': ttl,
             'type': _type,
             'values': [{
-                'priority': r[1],
-                'value': '{}.'.format(r[0])
+                'preference': r[1],
+                'exchange': '{}.'.format(r[0])
             } for r in records]
         }
 
@@ -80,19 +81,21 @@ class TinyDnsBaseSource(BaseSource):
             'values': ['{}.'.format(r[0]) for r in records]
         }
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: zone=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+
         before = len(zone.records)
 
         if zone.name.endswith('in-addr.arpa.'):
-            self._populate_in_addr_arpa(zone)
+            self._populate_in_addr_arpa(zone, lenient)
         else:
-            self._populate_normal(zone)
+            self._populate_normal(zone, lenient)
 
         self.log.info('populate:   found %s records',
                       len(zone.records) - before)
 
-    def _populate_normal(self, zone):
+    def _populate_normal(self, zone, lenient):
         type_map = {
             '=': 'A',
             '^': None,
@@ -128,14 +131,15 @@ class TinyDnsBaseSource(BaseSource):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 data = data_for(_type, d)
                 if data:
-                    record = Record.new(zone, name, data, source=self)
+                    record = Record.new(zone, name, data, source=self,
+                                        lenient=lenient)
                     try:
                         zone.add_record(record)
                     except SubzoneRecordException:
                         self.log.debug('_populate_normal: skipping subzone '
                                        'record=%s', record)
 
-    def _populate_in_addr_arpa(self, zone):
+    def _populate_in_addr_arpa(self, zone, lenient):
         name_re = re.compile('(?P<name>.+)\.{}$'.format(zone.name[:-1]))
 
         for line in self._lines():
@@ -169,7 +173,7 @@ class TinyDnsBaseSource(BaseSource):
                     'ttl': ttl,
                     'type': 'PTR',
                     'value': value
-                }, source=self)
+                }, source=self, lenient=lenient)
                 try:
                     zone.add_record(record)
                 except DuplicateRecordException:

octodns/yaml.py

@@ -21,9 +21,13 @@ class SortEnforcingLoader(SafeLoader):
         self.flatten_mapping(node)
         ret = self.construct_pairs(node)
         keys = [d[0] for d in ret]
-        if keys != sorted(keys, key=_natsort_key):
-            raise ConstructorError(None, None, "keys out of order: {}"
-                                   .format(', '.join(keys)), node.start_mark)
+        keys_sorted = sorted(keys, key=_natsort_key)
+        for key in keys:
+            expected = keys_sorted.pop(0)
+            if key != expected:
+                raise ConstructorError(None, None, 'keys out of order: '
+                                       'expected {} got {} at {}'
+                                       .format(expected, key, node.start_mark))
         return dict(ret)
 
 

octodns/zone.py

@@ -5,6 +5,7 @@
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
+from collections import defaultdict
 from logging import getLogger
 import re
 
@@ -19,6 +20,10 @@ class DuplicateRecordException(Exception):
     pass
 
 
+class InvalidNodeException(Exception):
+    pass
+
+
 def _is_eligible(record):
     # Should this record be considered when computing changes
     # We ignore all top-level NS records
@@ -35,19 +40,26 @@ class Zone(object):
         # Force everyting to lowercase just to be safe
         self.name = str(name).lower() if name else name
         self.sub_zones = sub_zones
-        self.records = set()
+        # We're grouping by node, it allows us to efficently search for
+        # duplicates and detect when CNAMEs co-exist with other records
+        self._records = defaultdict(set)
         # optional leading . to match empty hostname
         # optional trailing . b/c some sources don't have it on their fqdn
         self._name_re = re.compile('\.?{}?$'.format(name))
 
         self.log.debug('__init__: zone=%s, sub_zones=%s', self, sub_zones)
 
+    @property
+    def records(self):
+        return set([r for _, node in self._records.items() for r in node])
+
     def hostname_from_fqdn(self, fqdn):
         return self._name_re.sub('', fqdn)
 
-    def add_record(self, record):
+    def add_record(self, record, replace=False):
         name = record.name
         last = name.split('.')[-1]
+
         if last in self.sub_zones:
             if name != last:
                 # it's a record for something under a sub-zone
@@ -59,10 +71,30 @@ class Zone(object):
                 raise SubzoneRecordException('Record {} a managed sub-zone '
                                              'and not of type NS'
                                              .format(record.fqdn))
-        if record in self.records:
+
+        if replace:
+            # will remove it if it exists
+            self._records[name].discard(record)
+
+        node = self._records[name]
+        if record in node:
+            # We already have a record at this node of this type
             raise DuplicateRecordException('Duplicate record {}, type {}'
-                                           .format(record.fqdn, record._type))
-        self.records.add(record)
+                                           .format(record.fqdn,
+                                                   record._type))
+        elif ((record._type == 'CNAME' and len(node) > 0) or
+              ('CNAME' in map(lambda r: r._type, node))):
+            # We're adding a CNAME to existing records or adding to an existing
+            # CNAME
+            raise InvalidNodeException('Invalid state, CNAME at {} cannot '
+                                       'coexist with other records'
+                                       .format(record.fqdn))
+
+        node.add(record)
+
+    def _remove_record(self, record):
+        'Only for use in tests'
+        self._records[record.name].discard(record)
 
     def changes(self, desired, target):
         self.log.debug('changes: zone=%s, target=%s', self, target)

requirements-dev.txt

@@ -4,3 +4,4 @@ nose
 pep8
 pyflakes
 requests_mock
+setuptools>=36.4.0

requirements.txt

@@ -1,18 +1,23 @@
 # These are known good versions. You're free to use others and things will
 # likely work, but no promises are made, especilly if you go older.
 PyYaml==3.12
-boto3==1.4.4
-botocore==1.5.4
+azure-mgmt-dns==1.0.1
+azure-common==1.1.6
+boto3==1.4.6
+botocore==1.6.8
 dnspython==1.15.0
-docutils==0.13.1
-dyn==1.7.10
-futures==3.0.5
+docutils==0.14
+dyn==1.8.0
+futures==3.1.1
+google-cloud==0.27.0
 incf.countryutils==1.0
 ipaddress==1.0.18
-jmespath==0.9.0
+jmespath==0.9.3
+msrestazure==0.4.10
 natsort==5.0.3
-nsone==0.9.10
-python-dateutil==2.6.0
+nsone==0.9.14
+ovh==0.4.7
+python-dateutil==2.6.1
 requests==2.13.0
 s3transfer==0.1.10
 six==1.10.0

script/coverage

@@ -24,6 +24,7 @@ export DNSIMPLE_TOKEN=
 export DYN_CUSTOMER=
 export DYN_PASSWORD=
 export DYN_USERNAME=
+export GOOGLE_APPLICATION_CREDENTIALS=
 
 coverage run --branch --source=octodns `which nosetests` --with-xunit "$@"
 coverage html

script/release

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+cd "$(dirname $0)"/..
+ROOT=$(pwd)
+
+VERSION=$(grep __VERSION__ $ROOT/octodns/__init__.py | sed -e "s/.* = '//" -e "s/'$//")
+
+git tag -s v$VERSION -m "Release $VERSION"
+git push origin v$VERSION
+echo "Tagged and pushed v$VERSION"
+python setup.py sdist upload
+echo "Updloaded $VERSION"

script/test

@@ -24,5 +24,6 @@ export DNSIMPLE_TOKEN=
 export DYN_CUSTOMER=
 export DYN_PASSWORD=
 export DYN_USERNAME=
+export GOOGLE_APPLICATION_CREDENTIALS=
 
 nosetests "$@"

tests/config/unit.tests.yaml

@@ -31,6 +31,11 @@
     values:
     - 6.2.3.4.
     - 7.2.3.4.
+  - type: CAA
+    values:
+    - flags: 0
+      tag: issue
+      value: ca.unit.tests
 _srv._tcp:
   ttl: 600
   type: SRV
@@ -60,12 +65,12 @@ mx:
   ttl: 300
   type: MX
   values:
-  - priority: 40
-    value: smtp-1.unit.tests.
-  - priority: 20
-    value: smtp-2.unit.tests.
-  - priority: 30
-    value: smtp-3.unit.tests.
+  - exchange: smtp-1.unit.tests.
+    preference: 40
+  - exchange: smtp-2.unit.tests.
+    preference: 20
+  - exchange: smtp-3.unit.tests.
+    preference: 30
   - priority: 10
     value: smtp-4.unit.tests.
 naptr:

tests/fixtures/cloudflare-dns_records-page-2.json

@@ -118,14 +118,35 @@
       "meta": {
         "auto_added": false
       }
+    },
+    {
+      "id": "fc223b34cd5611334422ab3322997667",
+      "type": "CAA",
+      "name": "unit.tests",
+      "data": {
+          "flags": 0,
+          "tag": "issue",
+          "value": "ca.unit.tests"
+      },
+      "proxiable": false,
+      "proxied": false,
+      "ttl": 3600,
+      "locked": false,
+      "zone_id": "ff12ab34cd5611334422ab3322997650",
+      "zone_name": "unit.tests",
+      "modified_on": "2017-03-11T18:01:42.961566Z",
+      "created_on": "2017-03-11T18:01:42.961566Z",
+      "meta": {
+        "auto_added": false
+      }
     }
   ],
   "result_info": {
     "page": 2,
     "per_page": 10,
     "total_pages": 2,
-    "count": 7,
-    "total_count": 17
+    "count": 8,
+    "total_count": 19
   },
   "success": true,
   "errors": [],

tests/fixtures/dnsimple-page-2.json

@@ -159,12 +159,28 @@
       "system_record": false,
       "created_at": "2017-03-09T15:55:09Z",
       "updated_at": "2017-03-09T15:55:09Z"
+    },
+    {
+      "id": 12188803,
+      "zone_id": "unit.tests",
+      "parent_id": null,
+      "name": "",
+      "content": "0 issue \"ca.unit.tests\"",
+      "ttl": 3600,
+      "priority": null,
+      "type": "CAA",
+      "regions": [
+        "global"
+      ],
+      "system_record": false,
+      "created_at": "2017-03-09T15:55:09Z",
+      "updated_at": "2017-03-09T15:55:09Z"
     }
   ],
   "pagination": {
     "current_page": 2,
     "per_page": 20,
-    "total_entries": 29,
+    "total_entries": 30,
     "total_pages": 2
   }
 }

tests/fixtures/powerdns-full-data.json

@@ -230,6 +230,18 @@
             ],
             "ttl": 300,
             "type": "A"
+        },
+        {
+            "comments": [],
+            "name": "unit.tests.",
+            "records": [
+                {
+                    "content": "0 issue \"ca.unit.tests\"",
+                    "disabled": false
+                }
+            ],
+            "ttl": 3600,
+            "type": "CAA"
         }
     ],
     "serial": 2017012803,

tests/helpers.py

@@ -17,11 +17,12 @@ class SimpleSource(object):
 
 class SimpleProvider(object):
     SUPPORTS_GEO = False
+    SUPPORTS = set(('A',))
 
     def __init__(self, id='test'):
         pass
 
-    def populate(self, zone, source=True):
+    def populate(self, zone, source=False, lenient=False):
         pass
 
     def supports(self, record):
@@ -37,7 +38,7 @@ class GeoProvider(object):
     def __init__(self, id='test'):
         pass
 
-    def populate(self, zone, source=True):
+    def populate(self, zone, source=False, lenient=False):
         pass
 
     def supports(self, record):

tests/test_octodns_manager.py

@@ -11,6 +11,7 @@ from unittest import TestCase
 
 from octodns.record import Record
 from octodns.manager import _AggregateTarget, MainThreadExecutor, Manager
+from octodns.yaml import safe_load
 from octodns.zone import Zone
 
 from helpers import GeoProvider, NoSshFpProvider, SimpleProvider, \
@@ -128,6 +129,12 @@ class TestManager(TestCase):
                 .sync(dry_run=False, force=True)
             self.assertEquals(19, tc)
 
+            # Include meta
+            tc = Manager(get_config_filename('simple.yaml'), max_workers=1,
+                         include_meta=True) \
+                .sync(dry_run=False, force=True)
+            self.assertEquals(23, tc)
+
     def test_eligible_targets(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
@@ -195,15 +202,26 @@ class TestManager(TestCase):
             manager = Manager(get_config_filename('simple.yaml'))
 
             with self.assertRaises(Exception) as ctx:
-                manager.dump('unit.tests.', tmpdir.dirname, 'nope')
+                manager.dump('unit.tests.', tmpdir.dirname, False, 'nope')
             self.assertEquals('Unknown source: nope', ctx.exception.message)
 
-            manager.dump('unit.tests.', tmpdir.dirname, 'in')
+            manager.dump('unit.tests.', tmpdir.dirname, False, 'in')
 
             # make sure this fails with an IOError and not a KeyError when
             # tyring to find sub zones
             with self.assertRaises(IOError):
-                manager.dump('unknown.zone.', tmpdir.dirname, 'in')
+                manager.dump('unknown.zone.', tmpdir.dirname, False, 'in')
+
+    def test_dump_empty(self):
+        with TemporaryDirectory() as tmpdir:
+            environ['YAML_TMP_DIR'] = tmpdir.dirname
+            manager = Manager(get_config_filename('simple.yaml'))
+
+            manager.dump('empty.', tmpdir.dirname, False, 'in')
+
+            with open(join(tmpdir.dirname, 'empty.yaml')) as fh:
+                data = safe_load(fh, False)
+                self.assertFalse(data)
 
     def test_validate_configs(self):
         Manager(get_config_filename('simple-validate.yaml')).validate_configs()

tests/test_octodns_provider_azuredns.py

@@ -0,0 +1,379 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from octodns.record import Create, Delete, Record
+from octodns.provider.azuredns import _AzureRecord, AzureProvider, \
+    _check_endswith_dot, _parse_azure_type
+from octodns.zone import Zone
+from octodns.provider.base import Plan
+
+from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \
+    SrvRecord, NsRecord, PtrRecord, TxtRecord, RecordSet, SoaRecord, \
+    Zone as AzureZone
+from msrestazure.azure_exceptions import CloudError
+
+from unittest import TestCase
+from mock import Mock, patch
+
+
+zone = Zone(name='unit.tests.', sub_zones=[])
+octo_records = []
+octo_records.append(Record.new(zone, '', {
+    'ttl': 0,
+    'type': 'A',
+    'values': ['1.2.3.4', '10.10.10.10']}))
+octo_records.append(Record.new(zone, 'a', {
+    'ttl': 1,
+    'type': 'A',
+    'values': ['1.2.3.4', '1.1.1.1']}))
+octo_records.append(Record.new(zone, 'aa', {
+    'ttl': 9001,
+    'type': 'A',
+    'values': ['1.2.4.3']}))
+octo_records.append(Record.new(zone, 'aaa', {
+    'ttl': 2,
+    'type': 'A',
+    'values': ['1.1.1.3']}))
+octo_records.append(Record.new(zone, 'cname', {
+    'ttl': 3,
+    'type': 'CNAME',
+    'value': 'a.unit.tests.'}))
+octo_records.append(Record.new(zone, 'mx1', {
+    'ttl': 3,
+    'type': 'MX',
+    'values': [{
+        'priority': 10,
+        'value': 'mx1.unit.tests.',
+    }, {
+        'priority': 20,
+        'value': 'mx2.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, 'mx2', {
+    'ttl': 3,
+    'type': 'MX',
+    'values': [{
+        'priority': 10,
+        'value': 'mx1.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, '', {
+    'ttl': 4,
+    'type': 'NS',
+    'values': ['ns1.unit.tests.', 'ns2.unit.tests.']}))
+octo_records.append(Record.new(zone, 'foo', {
+    'ttl': 5,
+    'type': 'NS',
+    'value': 'ns1.unit.tests.'}))
+octo_records.append(Record.new(zone, '_srv._tcp', {
+    'ttl': 6,
+    'type': 'SRV',
+    'values': [{
+        'priority': 10,
+        'weight': 20,
+        'port': 30,
+        'target': 'foo-1.unit.tests.',
+    }, {
+        'priority': 12,
+        'weight': 30,
+        'port': 30,
+        'target': 'foo-2.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, '_srv2._tcp', {
+    'ttl': 7,
+    'type': 'SRV',
+    'values': [{
+        'priority': 12,
+        'weight': 17,
+        'port': 1,
+        'target': 'srvfoo.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, 'txt1', {
+    'ttl': 8,
+    'type': 'TXT',
+    'value': 'txt singleton test'}))
+octo_records.append(Record.new(zone, 'txt2', {
+    'ttl': 9,
+    'type': 'TXT',
+    'values': ['txt multiple test', 'txt multiple test 2']}))
+
+azure_records = []
+_base0 = _AzureRecord('TestAzure', octo_records[0])
+_base0.zone_name = 'unit.tests'
+_base0.relative_record_set_name = '@'
+_base0.record_type = 'A'
+_base0.params['ttl'] = 0
+_base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')]
+azure_records.append(_base0)
+
+_base1 = _AzureRecord('TestAzure', octo_records[1])
+_base1.zone_name = 'unit.tests'
+_base1.relative_record_set_name = 'a'
+_base1.record_type = 'A'
+_base1.params['ttl'] = 1
+_base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')]
+azure_records.append(_base1)
+
+_base2 = _AzureRecord('TestAzure', octo_records[2])
+_base2.zone_name = 'unit.tests'
+_base2.relative_record_set_name = 'aa'
+_base2.record_type = 'A'
+_base2.params['ttl'] = 9001
+_base2.params['arecords'] = ARecord('1.2.4.3')
+azure_records.append(_base2)
+
+_base3 = _AzureRecord('TestAzure', octo_records[3])
+_base3.zone_name = 'unit.tests'
+_base3.relative_record_set_name = 'aaa'
+_base3.record_type = 'A'
+_base3.params['ttl'] = 2
+_base3.params['arecords'] = ARecord('1.1.1.3')
+azure_records.append(_base3)
+
+_base4 = _AzureRecord('TestAzure', octo_records[4])
+_base4.zone_name = 'unit.tests'
+_base4.relative_record_set_name = 'cname'
+_base4.record_type = 'CNAME'
+_base4.params['ttl'] = 3
+_base4.params['cname_record'] = CnameRecord('a.unit.tests.')
+azure_records.append(_base4)
+
+_base5 = _AzureRecord('TestAzure', octo_records[5])
+_base5.zone_name = 'unit.tests'
+_base5.relative_record_set_name = 'mx1'
+_base5.record_type = 'MX'
+_base5.params['ttl'] = 3
+_base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'),
+                               MxRecord(20, 'mx2.unit.tests.')]
+azure_records.append(_base5)
+
+_base6 = _AzureRecord('TestAzure', octo_records[6])
+_base6.zone_name = 'unit.tests'
+_base6.relative_record_set_name = 'mx2'
+_base6.record_type = 'MX'
+_base6.params['ttl'] = 3
+_base6.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.')]
+azure_records.append(_base6)
+
+_base7 = _AzureRecord('TestAzure', octo_records[7])
+_base7.zone_name = 'unit.tests'
+_base7.relative_record_set_name = '@'
+_base7.record_type = 'NS'
+_base7.params['ttl'] = 4
+_base7.params['ns_records'] = [NsRecord('ns1.unit.tests.'),
+                               NsRecord('ns2.unit.tests.')]
+azure_records.append(_base7)
+
+_base8 = _AzureRecord('TestAzure', octo_records[8])
+_base8.zone_name = 'unit.tests'
+_base8.relative_record_set_name = 'foo'
+_base8.record_type = 'NS'
+_base8.params['ttl'] = 5
+_base8.params['ns_records'] = [NsRecord('ns1.unit.tests.')]
+azure_records.append(_base8)
+
+_base9 = _AzureRecord('TestAzure', octo_records[9])
+_base9.zone_name = 'unit.tests'
+_base9.relative_record_set_name = '_srv._tcp'
+_base9.record_type = 'SRV'
+_base9.params['ttl'] = 6
+_base9.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'),
+                                SrvRecord(12, 30, 30, 'foo-2.unit.tests.')]
+azure_records.append(_base9)
+
+_base10 = _AzureRecord('TestAzure', octo_records[10])
+_base10.zone_name = 'unit.tests'
+_base10.relative_record_set_name = '_srv2._tcp'
+_base10.record_type = 'SRV'
+_base10.params['ttl'] = 7
+_base10.params['srv_records'] = [SrvRecord(12, 17, 1, 'srvfoo.unit.tests.')]
+azure_records.append(_base10)
+
+_base11 = _AzureRecord('TestAzure', octo_records[11])
+_base11.zone_name = 'unit.tests'
+_base11.relative_record_set_name = 'txt1'
+_base11.record_type = 'TXT'
+_base11.params['ttl'] = 8
+_base11.params['txt_records'] = [TxtRecord(['txt singleton test'])]
+azure_records.append(_base11)
+
+_base12 = _AzureRecord('TestAzure', octo_records[12])
+_base12.zone_name = 'unit.tests'
+_base12.relative_record_set_name = 'txt2'
+_base12.record_type = 'TXT'
+_base12.params['ttl'] = 9
+_base12.params['txt_records'] = [TxtRecord(['txt multiple test']),
+                                 TxtRecord(['txt multiple test 2'])]
+azure_records.append(_base12)
+
+
+class Test_AzureRecord(TestCase):
+    def test_azure_record(self):
+        assert(len(azure_records) == len(octo_records))
+        for i in range(len(azure_records)):
+            octo = _AzureRecord('TestAzure', octo_records[i])
+            assert(azure_records[i]._equals(octo))
+
+
+class Test_ParseAzureType(TestCase):
+    def test_parse_azure_type(self):
+        for expected, test in [['A', 'Microsoft.Network/dnszones/A'],
+                               ['AAAA', 'Microsoft.Network/dnszones/AAAA'],
+                               ['NS', 'Microsoft.Network/dnszones/NS'],
+                               ['MX', 'Microsoft.Network/dnszones/MX']]:
+                self.assertEquals(expected, _parse_azure_type(test))
+
+
+class Test_CheckEndswithDot(TestCase):
+    def test_check_endswith_dot(self):
+        for expected, test in [['a.', 'a'],
+                               ['a.', 'a.'],
+                               ['foo.bar.', 'foo.bar.'],
+                               ['foo.bar.', 'foo.bar']]:
+            self.assertEquals(expected, _check_endswith_dot(test))
+
+
+class TestAzureDnsProvider(TestCase):
+    def _provider(self):
+        return self._get_provider('mock_spc', 'mock_dns_client')
+
+    @patch('octodns.provider.azuredns.DnsManagementClient')
+    @patch('octodns.provider.azuredns.ServicePrincipalCredentials')
+    def _get_provider(self, mock_spc, mock_dns_client):
+        '''Returns a mock AzureProvider object to use in testing.
+
+            :param mock_spc: placeholder
+            :type  mock_spc: str
+            :param mock_dns_client: placeholder
+            :type  mock_dns_client: str
+
+            :type return: AzureProvider
+        '''
+        return AzureProvider('mock_id', 'mock_client', 'mock_key',
+                             'mock_directory', 'mock_sub', 'mock_rg')
+
+    def test_populate_records(self):
+        provider = self._get_provider()
+
+        rs = []
+        rs.append(RecordSet(name='a1', ttl=0, type='A',
+                            arecords=[ARecord('1.1.1.1')]))
+        rs.append(RecordSet(name='a2', ttl=1, type='A',
+                            arecords=[ARecord('1.1.1.1'),
+                                      ARecord('2.2.2.2')]))
+        rs.append(RecordSet(name='aaaa1', ttl=2, type='AAAA',
+                            aaaa_records=[AaaaRecord('1:1ec:1::1')]))
+        rs.append(RecordSet(name='aaaa2', ttl=3, type='AAAA',
+                            aaaa_records=[AaaaRecord('1:1ec:1::1'),
+                                          AaaaRecord('1:1ec:1::2')]))
+        rs.append(RecordSet(name='cname1', ttl=4, type='CNAME',
+                            cname_record=CnameRecord('cname.unit.test.')))
+        rs.append(RecordSet(name='cname2', ttl=5, type='CNAME',
+                            cname_record=None))
+        rs.append(RecordSet(name='mx1', ttl=6, type='MX',
+                            mx_records=[MxRecord(10, 'mx1.unit.test.')]))
+        rs.append(RecordSet(name='mx2', ttl=7, type='MX',
+                            mx_records=[MxRecord(10, 'mx1.unit.test.'),
+                                        MxRecord(11, 'mx2.unit.test.')]))
+        rs.append(RecordSet(name='ns1', ttl=8, type='NS',
+                            ns_records=[NsRecord('ns1.unit.test.')]))
+        rs.append(RecordSet(name='ns2', ttl=9, type='NS',
+                            ns_records=[NsRecord('ns1.unit.test.'),
+                                        NsRecord('ns2.unit.test.')]))
+        rs.append(RecordSet(name='ptr1', ttl=10, type='PTR',
+                            ptr_records=[PtrRecord('ptr1.unit.test.')]))
+        rs.append(RecordSet(name='ptr2', ttl=11, type='PTR',
+                            ptr_records=[PtrRecord(None)]))
+        rs.append(RecordSet(name='_srv1._tcp', ttl=12, type='SRV',
+                            srv_records=[SrvRecord(1, 2, 3, '1unit.tests.')]))
+        rs.append(RecordSet(name='_srv2._tcp', ttl=13, type='SRV',
+                            srv_records=[SrvRecord(1, 2, 3, '1unit.tests.'),
+                                         SrvRecord(4, 5, 6, '2unit.tests.')]))
+        rs.append(RecordSet(name='txt1', ttl=14, type='TXT',
+                            txt_records=[TxtRecord('sample text1')]))
+        rs.append(RecordSet(name='txt2', ttl=15, type='TXT',
+                            txt_records=[TxtRecord('sample text1'),
+                                         TxtRecord('sample text2')]))
+        rs.append(RecordSet(name='', ttl=16, type='SOA',
+                            soa_record=[SoaRecord()]))
+
+        record_list = provider._dns_client.record_sets.list_by_dns_zone
+        record_list.return_value = rs
+
+        provider.populate(zone)
+
+        self.assertEquals(len(zone.records), 16)
+
+    def test_populate_zone(self):
+        provider = self._get_provider()
+
+        zone_list = provider._dns_client.zones.list_by_resource_group
+        zone_list.return_value = [AzureZone(location='global'),
+                                  AzureZone(location='global')]
+
+        provider._populate_zones()
+
+        self.assertEquals(len(provider._azure_zones), 1)
+
+    def test_bad_zone_response(self):
+        provider = self._get_provider()
+
+        _get = provider._dns_client.zones.get
+        _get.side_effect = CloudError(Mock(status=404), 'Azure Error')
+        trip = False
+        try:
+            provider._check_zone('unit.test', create=False)
+        except CloudError:
+            trip = True
+        self.assertEquals(trip, True)
+
+    def test_apply(self):
+        provider = self._get_provider()
+
+        changes = []
+        deletes = []
+        for i in octo_records:
+            changes.append(Create(i))
+            deletes.append(Delete(i))
+
+        self.assertEquals(13, provider.apply(Plan(None, zone, changes)))
+        self.assertEquals(13, provider.apply(Plan(zone, zone, deletes)))
+
+    def test_create_zone(self):
+        provider = self._get_provider()
+
+        changes = []
+        for i in octo_records:
+            changes.append(Create(i))
+        desired = Zone('unit2.test.', [])
+
+        err_msg = 'The Resource \'Microsoft.Network/dnszones/unit2.test\' '
+        err_msg += 'under resource group \'mock_rg\' was not found.'
+        _get = provider._dns_client.zones.get
+        _get.side_effect = CloudError(Mock(status=404), err_msg)
+
+        self.assertEquals(13, provider.apply(Plan(None, desired, changes)))
+
+    def test_check_zone_no_create(self):
+        provider = self._get_provider()
+
+        rs = []
+        rs.append(RecordSet(name='a1', ttl=0, type='A',
+                            arecords=[ARecord('1.1.1.1')]))
+        rs.append(RecordSet(name='a2', ttl=1, type='A',
+                            arecords=[ARecord('1.1.1.1'),
+                                      ARecord('2.2.2.2')]))
+
+        record_list = provider._dns_client.record_sets.list_by_dns_zone
+        record_list.return_value = rs
+
+        err_msg = 'The Resource \'Microsoft.Network/dnszones/unit3.test\' '
+        err_msg += 'under resource group \'mock_rg\' was not found.'
+        _get = provider._dns_client.zones.get
+        _get.side_effect = CloudError(Mock(status=404), err_msg)
+
+        provider.populate(Zone('unit3.test.', []))
+
+        self.assertEquals(len(zone.records), 0)

tests/test_octodns_provider_base.py

@@ -16,13 +16,17 @@ from octodns.zone import Zone
 class HelperProvider(BaseProvider):
     log = getLogger('HelperProvider')
 
+    SUPPORTS = set(('A',))
+
     def __init__(self, extra_changes, apply_disabled=False,
                  include_change_callback=None):
         self.__extra_changes = extra_changes
         self.apply_disabled = apply_disabled
         self.include_change_callback = include_change_callback
+        self.update_pcent_threshold = Plan.MAX_SAFE_UPDATE_PCENT
+        self.delete_pcent_threshold = Plan.MAX_SAFE_DELETE_PCENT
 
-    def populate(self, zone, target=False):
+    def populate(self, zone, target=False, lenient=False):
         pass
 
     def _include_change(self, change):
@@ -58,12 +62,19 @@ class TestBaseProvider(TestCase):
         zone = Zone('unit.tests.', [])
         with self.assertRaises(NotImplementedError) as ctx:
             HasSupportsGeo('hassupportesgeo').populate(zone)
+        self.assertEquals('Abstract base class, SUPPORTS property missing',
+                          ctx.exception.message)
+
+        class HasSupports(HasSupportsGeo):
+            SUPPORTS = set(('A',))
+        with self.assertRaises(NotImplementedError) as ctx:
+            HasSupports('hassupportes').populate(zone)
         self.assertEquals('Abstract base class, populate method missing',
                           ctx.exception.message)
 
-        class HasPopulate(HasSupportsGeo):
+        class HasPopulate(HasSupports):
 
-            def populate(self, zone, target=False):
+            def populate(self, zone, target=False, lenient=False):
                 zone.add_record(Record.new(zone, '', {
                     'ttl': 60,
                     'type': 'A',
@@ -81,7 +92,7 @@ class TestBaseProvider(TestCase):
             'value': '1.2.3.4'
         }))
 
-        self.assertTrue(HasSupportsGeo('hassupportesgeo')
+        self.assertTrue(HasSupports('hassupportesgeo')
                         .supports(list(zone.records)[0]))
 
         plan = HasPopulate('haspopulate').plan(zone)
@@ -205,9 +216,11 @@ class TestBaseProvider(TestCase):
                    for i in range(int(Plan.MIN_EXISTING_RECORDS *
                                       Plan.MAX_SAFE_UPDATE_PCENT) + 1)]
 
-        with self.assertRaises(UnsafePlan):
+        with self.assertRaises(UnsafePlan) as ctx:
             Plan(zone, zone, changes).raise_if_unsafe()
 
+        self.assertTrue('Too many updates' in ctx.exception.message)
+
     def test_safe_updates_min_existing_pcent(self):
         # MAX_SAFE_UPDATE_PCENT is safe when more
         # than MIN_EXISTING_RECORDS exist
@@ -251,9 +264,11 @@ class TestBaseProvider(TestCase):
                    for i in range(int(Plan.MIN_EXISTING_RECORDS *
                                       Plan.MAX_SAFE_DELETE_PCENT) + 1)]
 
-        with self.assertRaises(UnsafePlan):
+        with self.assertRaises(UnsafePlan) as ctx:
             Plan(zone, zone, changes).raise_if_unsafe()
 
+        self.assertTrue('Too many deletes' in ctx.exception.message)
+
     def test_safe_deletes_min_existing_pcent(self):
         # MAX_SAFE_DELETE_PCENT is safe when more
         # than MIN_EXISTING_RECORDS exist
@@ -275,3 +290,59 @@ class TestBaseProvider(TestCase):
                                       Plan.MAX_SAFE_DELETE_PCENT))]
 
         Plan(zone, zone, changes).raise_if_unsafe()
+
+    def test_safe_updates_min_existing_override(self):
+        safe_pcent = .4
+        # 40% + 1 fails when more
+        # than MIN_EXISTING_RECORDS exist
+        zone = Zone('unit.tests.', [])
+        record = Record.new(zone, 'a', {
+            'ttl': 30,
+            'type': 'A',
+            'value': '1.2.3.4',
+        })
+
+        for i in range(int(Plan.MIN_EXISTING_RECORDS)):
+            zone.add_record(Record.new(zone, str(i), {
+                            'ttl': 60,
+                            'type': 'A',
+                            'value': '2.3.4.5'
+                            }))
+
+        changes = [Update(record, record)
+                   for i in range(int(Plan.MIN_EXISTING_RECORDS *
+                                      safe_pcent) + 1)]
+
+        with self.assertRaises(UnsafePlan) as ctx:
+            Plan(zone, zone, changes,
+                 update_pcent_threshold=safe_pcent).raise_if_unsafe()
+
+        self.assertTrue('Too many updates' in ctx.exception.message)
+
+    def test_safe_deletes_min_existing_override(self):
+        safe_pcent = .4
+        # 40% + 1 fails when more
+        # than MIN_EXISTING_RECORDS exist
+        zone = Zone('unit.tests.', [])
+        record = Record.new(zone, 'a', {
+            'ttl': 30,
+            'type': 'A',
+            'value': '1.2.3.4',
+        })
+
+        for i in range(int(Plan.MIN_EXISTING_RECORDS)):
+            zone.add_record(Record.new(zone, str(i), {
+                            'ttl': 60,
+                            'type': 'A',
+                            'value': '2.3.4.5'
+                            }))
+
+        changes = [Delete(record)
+                   for i in range(int(Plan.MIN_EXISTING_RECORDS *
+                                      safe_pcent) + 1)]
+
+        with self.assertRaises(UnsafePlan) as ctx:
+            Plan(zone, zone, changes,
+                 delete_pcent_threshold=safe_pcent).raise_if_unsafe()
+
+        self.assertTrue('Too many deletes' in ctx.exception.message)

tests/test_octodns_provider_cloudflare.py

@@ -33,7 +33,7 @@ class TestCloudflareProvider(TestCase):
     }))
     for record in list(expected.records):
         if record.name == 'sub' and record._type == 'NS':
-            expected.records.remove(record)
+            expected._remove_record(record)
             break
 
     empty = {'result': [], 'result_info': {'count': 0, 'per_page': 0}}
@@ -118,7 +118,7 @@ class TestCloudflareProvider(TestCase):
 
             zone = Zone('unit.tests.', [])
             provider.populate(zone)
-            self.assertEquals(9, len(zone.records))
+            self.assertEquals(10, len(zone.records))
 
             changes = self.expected.changes(zone, provider)
             self.assertEquals(0, len(changes))
@@ -126,7 +126,7 @@ class TestCloudflareProvider(TestCase):
         # re-populating the same zone/records comes out of cache, no calls
         again = Zone('unit.tests.', [])
         provider.populate(again)
-        self.assertEquals(9, len(again.records))
+        self.assertEquals(10, len(again.records))
 
     def test_apply(self):
         provider = CloudflareProvider('test', 'email', 'token')
@@ -140,12 +140,12 @@ class TestCloudflareProvider(TestCase):
                     'id': 42,
                 }
             },  # zone create
-        ] + [None] * 16  # individual record creates
+        ] + [None] * 17  # individual record creates
 
         # non-existant zone, create everything
         plan = provider.plan(self.expected)
-        self.assertEquals(9, len(plan.changes))
-        self.assertEquals(9, provider.apply(plan))
+        self.assertEquals(10, len(plan.changes))
+        self.assertEquals(10, provider.apply(plan))
 
         provider._request.assert_has_calls([
             # created the domain
@@ -170,7 +170,7 @@ class TestCloudflareProvider(TestCase):
             }),
         ], True)
         # expected number of total calls
-        self.assertEquals(18, provider._request.call_count)
+        self.assertEquals(19, provider._request.call_count)
 
         provider._request.reset_mock()
 

tests/test_octodns_provider_dnsimple.py

@@ -33,7 +33,7 @@ class TestDnsimpleProvider(TestCase):
     }))
     for record in list(expected.records):
         if record.name == 'sub' and record._type == 'NS':
-            expected.records.remove(record)
+            expected._remove_record(record)
             break
 
     def test_populate(self):
@@ -78,14 +78,14 @@ class TestDnsimpleProvider(TestCase):
 
             zone = Zone('unit.tests.', [])
             provider.populate(zone)
-            self.assertEquals(14, len(zone.records))
+            self.assertEquals(15, len(zone.records))
             changes = self.expected.changes(zone, provider)
             self.assertEquals(0, len(changes))
 
         # 2nd populate makes no network calls/all from cache
         again = Zone('unit.tests.', [])
         provider.populate(again)
-        self.assertEquals(14, len(again.records))
+        self.assertEquals(15, len(again.records))
 
         # bust the cache
         del provider._zone_records[zone.name]
@@ -147,7 +147,7 @@ class TestDnsimpleProvider(TestCase):
             }),
         ])
         # expected number of total calls
-        self.assertEquals(26, provider._client._request.call_count)
+        self.assertEquals(27, provider._client._request.call_count)
 
         provider._client._request.reset_mock()
 

tests/test_octodns_provider_dyn.py

@@ -46,11 +46,11 @@ class TestDynProvider(TestCase):
             'type': 'MX',
             'ttl': 302,
             'values': [{
-                'priority': 10,
-                'value': 'smtp-1.unit.tests.'
+                'preference': 10,
+                'exchange': 'smtp-1.unit.tests.'
             }, {
-                'priority': 20,
-                'value': 'smtp-2.unit.tests.'
+                'preference': 20,
+                'exchange': 'smtp-2.unit.tests.'
             }]
         }),
         ('naptr', {
@@ -109,6 +109,14 @@ class TestDynProvider(TestCase):
                 'weight': 22,
                 'port': 20,
                 'target': 'foo-2.unit.tests.'
+            }]}),
+        ('', {
+            'type': 'CAA',
+            'ttl': 308,
+            'values': [{
+                'flags': 0,
+                'tag': 'issue',
+                'value': 'ca.unit.tests'
             }]})):
         expected.add_record(Record.new(expected, name, data))
 
@@ -321,6 +329,16 @@ class TestDynProvider(TestCase):
                     'ttl': 307,
                     'zone': 'unit.tests',
                 }],
+                'caa_records': [{
+                    'fqdn': 'unit.tests',
+                    'rdata': {'flags': 0,
+                              'tag': 'issue',
+                              'value': 'ca.unit.tests'},
+                    'record_id': 12,
+                    'record_type': 'cAA',
+                    'ttl': 308,
+                    'zone': 'unit.tests',
+                }],
             }}
         ]
         got = Zone('unit.tests.', [])
@@ -414,10 +432,10 @@ class TestDynProvider(TestCase):
                 update_mock.assert_called()
             add_mock.assert_called()
             # Once for each dyn record (8 Records, 2 of which have dual values)
-            self.assertEquals(14, len(add_mock.call_args_list))
+            self.assertEquals(15, len(add_mock.call_args_list))
         execute_mock.assert_has_calls([call('/Zone/unit.tests/', 'GET', {}),
                                        call('/Zone/unit.tests/', 'GET', {})])
-        self.assertEquals(9, len(plan.changes))
+        self.assertEquals(10, len(plan.changes))
 
         execute_mock.reset_mock()
 
@@ -602,6 +620,7 @@ class TestDynProviderGeo(TestCase):
         provider = DynProvider('test', 'cust', 'user', 'pass', True)
         # short-circuit session checking
         provider._dyn_sess = True
+        provider.log.warn = MagicMock()
 
         # no tds
         mock.side_effect = [{'data': []}]
@@ -650,6 +669,10 @@ class TestDynProviderGeo(TestCase):
                           set(tds.keys()))
         self.assertEquals(['A'], tds['unit.tests.'].keys())
         self.assertEquals(['A'], tds['geo.unit.tests.'].keys())
+        provider.log.warn.assert_called_with("Failed to load TraficDirector "
+                                             "'%s': %s", 'something else',
+                                             'need more than 1 value to '
+                                             'unpack')
 
     @patch('dyn.core.SessionEngine.execute')
     def test_traffic_director_monitor(self, mock):

tests/test_octodns_provider_googlecloud.py

@@ -0,0 +1,429 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from octodns.record import Create, Delete, Update, Record
+from octodns.provider.googlecloud import GoogleCloudProvider
+
+from octodns.zone import Zone
+from octodns.provider.base import Plan, BaseProvider
+
+from unittest import TestCase
+from mock import Mock, patch, PropertyMock
+
+zone = Zone(name='unit.tests.', sub_zones=[])
+octo_records = []
+octo_records.append(Record.new(zone, '', {
+    'ttl': 0,
+    'type': 'A',
+    'values': ['1.2.3.4', '10.10.10.10']}))
+octo_records.append(Record.new(zone, 'a', {
+    'ttl': 1,
+    'type': 'A',
+    'values': ['1.2.3.4', '1.1.1.1']}))
+octo_records.append(Record.new(zone, 'aa', {
+    'ttl': 9001,
+    'type': 'A',
+    'values': ['1.2.4.3']}))
+octo_records.append(Record.new(zone, 'aaa', {
+    'ttl': 2,
+    'type': 'A',
+    'values': ['1.1.1.3']}))
+octo_records.append(Record.new(zone, 'cname', {
+    'ttl': 3,
+    'type': 'CNAME',
+    'value': 'a.unit.tests.'}))
+octo_records.append(Record.new(zone, 'mx1', {
+    'ttl': 3,
+    'type': 'MX',
+    'values': [{
+        'priority': 10,
+        'value': 'mx1.unit.tests.',
+    }, {
+        'priority': 20,
+        'value': 'mx2.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, 'mx2', {
+    'ttl': 3,
+    'type': 'MX',
+    'values': [{
+        'priority': 10,
+        'value': 'mx1.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, '', {
+    'ttl': 4,
+    'type': 'NS',
+    'values': ['ns1.unit.tests.', 'ns2.unit.tests.']}))
+octo_records.append(Record.new(zone, 'foo', {
+    'ttl': 5,
+    'type': 'NS',
+    'value': 'ns1.unit.tests.'}))
+octo_records.append(Record.new(zone, '_srv._tcp', {
+    'ttl': 6,
+    'type': 'SRV',
+    'values': [{
+        'priority': 10,
+        'weight': 20,
+        'port': 30,
+        'target': 'foo-1.unit.tests.',
+    }, {
+        'priority': 12,
+        'weight': 30,
+        'port': 30,
+        'target': 'foo-2.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, '_srv2._tcp', {
+    'ttl': 7,
+    'type': 'SRV',
+    'values': [{
+        'priority': 12,
+        'weight': 17,
+        'port': 1,
+        'target': 'srvfoo.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, 'txt1', {
+    'ttl': 8,
+    'type': 'TXT',
+    'value': 'txt singleton test'}))
+octo_records.append(Record.new(zone, 'txt2', {
+    'ttl': 9,
+    'type': 'TXT',
+    'values': ['txt multiple test', 'txt multiple test 2']}))
+octo_records.append(Record.new(zone, 'naptr', {
+    'ttl': 9,
+    'type': 'NAPTR',
+    'values': [{
+        'order': 100,
+        'preference': 10,
+        'flags': 'S',
+        'service': 'SIP+D2U',
+        'regexp': "!^.*$!sip:[email protected]!",
+        'replacement': '_sip._udp.unit.tests.'
+    }]}))
+octo_records.append(Record.new(zone, 'caa', {
+    'ttl': 9,
+    'type': 'CAA',
+    'value': {
+        'flags': 0,
+        'tag': 'issue',
+        'value': 'ca.unit.tests',
+    }}))
+for record in octo_records:
+    zone.add_record(record)
+
+# This is the format which the google API likes.
+resource_record_sets = [
+    ('unit.tests.', u'A', 0, [u'1.2.3.4', u'10.10.10.10']),
+    (u'a.unit.tests.', u'A', 1, [u'1.1.1.1', u'1.2.3.4']),
+    (u'aa.unit.tests.', u'A', 9001, [u'1.2.4.3']),
+    (u'aaa.unit.tests.', u'A', 2, [u'1.1.1.3']),
+    (u'cname.unit.tests.', u'CNAME', 3, [u'a.unit.tests.']),
+    (u'mx1.unit.tests.', u'MX', 3,
+     [u'10 mx1.unit.tests.', u'20 mx2.unit.tests.']),
+    (u'mx2.unit.tests.', u'MX', 3, [u'10 mx1.unit.tests.']),
+    ('unit.tests.', u'NS', 4, [u'ns1.unit.tests.', u'ns2.unit.tests.']),
+    (u'foo.unit.tests.', u'NS', 5, [u'ns1.unit.tests.']),
+    (u'_srv._tcp.unit.tests.', u'SRV', 6,
+     [u'10 20 30 foo-1.unit.tests.', u'12 30 30 foo-2.unit.tests.']),
+    (u'_srv2._tcp.unit.tests.', u'SRV', 7, [u'12 17 1 srvfoo.unit.tests.']),
+    (u'txt1.unit.tests.', u'TXT', 8, [u'txt singleton test']),
+    (u'txt2.unit.tests.', u'TXT', 9,
+     [u'txt multiple test', u'txt multiple test 2']),
+    (u'naptr.unit.tests.', u'NAPTR', 9, [
+        u'100 10 "S" "SIP+D2U" "!^.*$!sip:[email protected]!"'
+        u' _sip._udp.unit.tests.']),
+    (u'caa.unit.tests.', u'CAA', 9, [u'0 issue ca.unit.tests'])
+]
+
+
+class DummyResourceRecordSet:
+    def __init__(self, record_name, record_type, ttl, rrdatas):
+        self.name = record_name
+        self.record_type = record_type
+        self.ttl = ttl
+        self.rrdatas = rrdatas
+
+    def __eq__(self, other):
+        try:
+            return self.name == other.name \
+                and self.record_type == other.record_type \
+                and self.ttl == other.ttl  \
+                and sorted(self.rrdatas) == sorted(other.rrdatas)
+        except:
+            return False
+
+    def __repr__(self):
+        return "{} {} {} {!s}"\
+            .format(self.name, self.record_type, self.ttl, self.rrdatas)
+
+    def __hash__(self):
+        return hash(repr(self))
+
+
+class DummyGoogleCloudZone:
+    def __init__(self, dns_name, name=""):
+        self.dns_name = dns_name
+        self.name = name
+
+    def resource_record_set(self, *args):
+        return DummyResourceRecordSet(*args)
+
+    def list_resource_record_sets(self, *args):
+        pass
+
+    def create(self, *args, **kwargs):
+        pass
+
+
+class DummyIterator:
+    """Returns a mock DummyIterator object to use in testing.
+    This is because API calls for google cloud DNS, if paged, contains a
+    "next_page_token", which can be used to grab a subsequent
+    iterator with more results.
+
+        :type return: DummyIterator
+    """
+    def __init__(self,  list_of_stuff, page_token=None):
+        self.iterable = iter(list_of_stuff)
+        self.next_page_token = page_token
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        return self.iterable.next()
+
+
+class TestGoogleCloudProvider(TestCase):
+    @patch('octodns.provider.googlecloud.dns')
+    def _get_provider(*args):
+        '''Returns a mock GoogleCloudProvider object to use in testing.
+
+            :type return: GoogleCloudProvider
+        '''
+        return GoogleCloudProvider(id=1, project="mock")
+
+    @patch('octodns.provider.googlecloud.dns')
+    def test___init__(self, *_):
+        self.assertIsInstance(GoogleCloudProvider(id=1,
+                                                  credentials_file="test",
+                                                  project="unit test"),
+                              BaseProvider)
+
+        self.assertIsInstance(GoogleCloudProvider(id=1),
+                              BaseProvider)
+
+    @patch('octodns.provider.googlecloud.time.sleep')
+    @patch('octodns.provider.googlecloud.dns')
+    def test__apply(self, *_):
+        class DummyDesired:
+            def __init__(self, name, changes):
+                self.name = name
+                self.changes = changes
+
+        apply_z = Zone("unit.tests.", [])
+        create_r = Record.new(apply_z, '', {
+            'ttl': 0,
+            'type': 'A',
+            'values': ['1.2.3.4', '10.10.10.10']})
+        delete_r = Record.new(apply_z, 'a', {
+            'ttl': 1,
+            'type': 'A',
+            'values': ['1.2.3.4', '1.1.1.1']})
+        update_existing_r = Record.new(apply_z, 'aa', {
+            'ttl': 9001,
+            'type': 'A',
+            'values': ['1.2.4.3']})
+        update_new_r = Record.new(apply_z, 'aa', {
+            'ttl': 666,
+            'type': 'A',
+            'values': ['1.4.3.2']})
+
+        gcloud_zone_mock = DummyGoogleCloudZone("unit.tests.", "unit-tests")
+        status_mock = Mock()
+        return_values_for_status = iter(
+            ["pending"] * 11 + ['done', 'done'])
+        type(status_mock).status = PropertyMock(
+            side_effect=return_values_for_status.next)
+        gcloud_zone_mock.changes = Mock(return_value=status_mock)
+
+        provider = self._get_provider()
+        provider.gcloud_client = Mock()
+        provider._gcloud_zones = {"unit.tests.": gcloud_zone_mock}
+        desired = Mock()
+        desired.name = "unit.tests."
+        changes = []
+        changes.append(Create(create_r))
+        changes.append(Delete(delete_r))
+        changes.append(Update(existing=update_existing_r, new=update_new_r))
+
+        provider.apply(Plan(
+            existing=[update_existing_r, delete_r],
+            desired=desired,
+            changes=changes
+        ))
+
+        calls_mock = gcloud_zone_mock.changes.return_value
+        mocked_calls = []
+        for mock_call in calls_mock.add_record_set.mock_calls:
+            mocked_calls.append(mock_call[1][0])
+
+        self.assertEqual(mocked_calls, [
+            DummyResourceRecordSet(
+                'unit.tests.', 'A', 0, ['1.2.3.4', '10.10.10.10']),
+            DummyResourceRecordSet(
+                'aa.unit.tests.', 'A', 666, ['1.4.3.2'])
+        ])
+
+        mocked_calls2 = []
+        for mock_call in calls_mock.delete_record_set.mock_calls:
+            mocked_calls2.append(mock_call[1][0])
+
+        self.assertEqual(mocked_calls2, [
+            DummyResourceRecordSet(
+                'a.unit.tests.', 'A', 1, ['1.2.3.4', '1.1.1.1']),
+            DummyResourceRecordSet(
+                'aa.unit.tests.', 'A', 9001, ['1.2.4.3'])
+        ])
+
+        type(status_mock).status = "pending"
+
+        with self.assertRaises(RuntimeError):
+            provider.apply(Plan(
+                existing=[update_existing_r, delete_r],
+                desired=desired,
+                changes=changes
+            ))
+
+        unsupported_change = Mock()
+        unsupported_change.__len__ = Mock(return_value=1)
+        type_mock = Mock()
+        type_mock._type = "A"
+        unsupported_change.record = type_mock
+
+        mock_plan = Mock()
+        type(mock_plan).desired = PropertyMock(return_value=DummyDesired(
+            "dummy name", []))
+        type(mock_plan).changes = [unsupported_change]
+
+        with self.assertRaises(RuntimeError):
+            provider.apply(mock_plan)
+
+    def test__get_gcloud_client(self):
+        provider = self._get_provider()
+
+        self.assertIsInstance(provider, GoogleCloudProvider)
+
+    @patch('octodns.provider.googlecloud.dns')
+    def test_populate(self, _):
+        def _get_mock_zones(page_token=None):
+            if not page_token:
+                return DummyIterator([
+                    DummyGoogleCloudZone('example.com.'),
+                ], page_token="MOCK_PAGE_TOKEN")
+            elif page_token == "MOCK_PAGE_TOKEN":
+                return DummyIterator([
+                    DummyGoogleCloudZone('example2.com.'),
+                ], page_token="MOCK_PAGE_TOKEN2")
+
+            return DummyIterator([
+                google_cloud_zone
+            ])
+
+        def _get_mock_record_sets(page_token=None):
+            if not page_token:
+                return DummyIterator(
+                    [DummyResourceRecordSet(*v) for v in
+                     resource_record_sets[:3]], page_token="MOCK_PAGE_TOKEN")
+            elif page_token == "MOCK_PAGE_TOKEN":
+
+                return DummyIterator(
+                    [DummyResourceRecordSet(*v) for v in
+                     resource_record_sets[3:5]], page_token="MOCK_PAGE_TOKEN2")
+            return DummyIterator(
+                [DummyResourceRecordSet(*v) for v in resource_record_sets[5:]])
+
+        google_cloud_zone = DummyGoogleCloudZone('unit.tests.')
+
+        provider = self._get_provider()
+        provider.gcloud_client.list_zones = Mock(side_effect=_get_mock_zones)
+        google_cloud_zone.list_resource_record_sets = Mock(
+            side_effect=_get_mock_record_sets)
+
+        self.assertEqual(provider.gcloud_zones.get("unit.tests.").dns_name,
+                         "unit.tests.")
+
+        test_zone = Zone('unit.tests.', [])
+        provider.populate(test_zone)
+
+        # test_zone gets fed the same records as zone does, except it's in
+        # the format returned by google API, so after populate they should look
+        # excactly the same.
+        self.assertEqual(test_zone.records, zone.records)
+
+        test_zone2 = Zone('nonexistant.zone.', [])
+        provider.populate(test_zone2, False, False)
+
+        self.assertEqual(len(test_zone2.records), 0,
+                         msg="Zone should not get records from wrong domain")
+
+        provider.SUPPORTS = set()
+        test_zone3 = Zone('unit.tests.', [])
+        provider.populate(test_zone3)
+        self.assertEqual(len(test_zone3.records), 0)
+
+    @patch('octodns.provider.googlecloud.dns')
+    def test_populate_corner_cases(self, _):
+        provider = self._get_provider()
+        test_zone = Zone('unit.tests.', [])
+        not_same_fqdn = DummyResourceRecordSet(
+            'unit.tests.gr', u'A', 0, [u'1.2.3.4']),
+
+        provider._get_gcloud_records = Mock(
+            side_effect=[not_same_fqdn])
+        provider._gcloud_zones = {
+            "unit.tests.": DummyGoogleCloudZone("unit.tests.", "unit-tests")}
+
+        provider.populate(test_zone)
+
+        self.assertEqual(len(test_zone.records), 1)
+
+        self.assertEqual(test_zone.records.pop().fqdn,
+                         u'unit.tests.gr.unit.tests.')
+
+    def test__get_gcloud_zone(self):
+        provider = self._get_provider()
+
+        provider.gcloud_client = Mock()
+        provider.gcloud_client.list_zones = Mock(
+            return_value=DummyIterator([]))
+
+        self.assertIsNone(provider.gcloud_zones.get("nonexistant.xone"),
+                          msg="Check that nonexistant zones return None when"
+                              "there's no create=True flag")
+
+    def test__get_rrsets(self):
+        provider = self._get_provider()
+        dummy_gcloud_zone = DummyGoogleCloudZone("unit.tests")
+        for octo_record in octo_records:
+            _rrset_func = getattr(
+                provider, '_rrset_for_{}'.format(octo_record._type))
+            self.assertEqual(
+                _rrset_func(dummy_gcloud_zone, octo_record).record_type,
+                octo_record._type
+            )
+
+    def test__create_zone(self):
+        provider = self._get_provider()
+
+        provider.gcloud_client = Mock()
+        provider.gcloud_client.list_zones = Mock(
+            return_value=DummyIterator([]))
+
+        mock_zone = provider._create_gcloud_zone("nonexistant.zone.mock")
+
+        mock_zone.create.assert_called()
+        provider.gcloud_client.zone.assert_called()

tests/test_octodns_provider_ns1.py

@@ -6,7 +6,8 @@ from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
 from mock import Mock, call, patch
-from nsone.rest.errors import AuthException, ResourceException
+from nsone.rest.errors import AuthException, RateLimitException, \
+    ResourceException
 from unittest import TestCase
 
 from octodns.record import Delete, Record, Update
@@ -44,11 +45,11 @@ class TestNs1Provider(TestCase):
         'ttl': 35,
         'type': 'MX',
         'values': [{
-            'priority': 10,
-            'value': 'mx1.unit.tests.',
+            'preference': 10,
+            'exchange': 'mx1.unit.tests.',
         }, {
-            'priority': 20,
-            'value': 'mx2.unit.tests.',
+            'preference': 20,
+            'exchange': 'mx2.unit.tests.',
         }]
     }))
     expected.add(Record.new(zone, 'naptr', {
@@ -95,6 +96,15 @@ class TestNs1Provider(TestCase):
         'type': 'NS',
         'values': ['ns3.unit.tests.', 'ns4.unit.tests.'],
     }))
+    expected.add(Record.new(zone, '', {
+        'ttl': 40,
+        'type': 'CAA',
+        'value': {
+            'flags': 0,
+            'tag': 'issue',
+            'value': 'ca.unit.tests',
+        },
+    }))
 
     nsone_records = [{
         'type': 'A',
@@ -140,6 +150,11 @@ class TestNs1Provider(TestCase):
         'ttl': 39,
         'short_answers': ['ns3.unit.tests.', 'ns4.unit.tests.'],
         'domain': 'sub.unit.tests.',
+    }, {
+        'type': 'CAA',
+        'ttl': 40,
+        'short_answers': ['0 issue ca.unit.tests'],
+        'domain': 'unit.tests.',
     }]
 
     @patch('nsone.NSONE.loadZone')
@@ -195,7 +210,8 @@ class TestNs1Provider(TestCase):
         provider = Ns1Provider('test', 'api-key')
 
         desired = Zone('unit.tests.', [])
-        desired.records.update(self.expected)
+        for r in self.expected:
+            desired.add_record(r)
 
         plan = provider.plan(desired)
         # everything except the root NS
@@ -225,7 +241,15 @@ class TestNs1Provider(TestCase):
         create_mock.reset_mock()
         load_mock.side_effect = \
             ResourceException('server error: zone not found')
-        create_mock.side_effect = None
+        # ugh, need a mock zone with a mock prop since we're using getattr, we
+        # can actually control side effects on `meth` with that.
+        mock_zone = Mock()
+        mock_zone.add_SRV = Mock()
+        mock_zone.add_SRV.side_effect = [
+            RateLimitException('boo', period=0),
+            None,
+        ]
+        create_mock.side_effect = [mock_zone]
         got_n = provider.apply(plan)
         self.assertEquals(expected_n, got_n)
 
@@ -245,12 +269,60 @@ class TestNs1Provider(TestCase):
         self.assertEquals(2, len(plan.changes))
         self.assertIsInstance(plan.changes[0], Update)
         self.assertIsInstance(plan.changes[1], Delete)
-
+        # ugh, we need a mock record that can be returned from loadRecord for
+        # the update and delete targets, we can add our side effects to that to
+        # trigger rate limit handling
+        mock_record = Mock()
+        mock_record.update.side_effect = [
+            RateLimitException('one', period=0),
+            None,
+        ]
+        mock_record.delete.side_effect = [
+            RateLimitException('two', period=0),
+            None,
+        ]
+        nsone_zone.loadRecord.side_effect = [mock_record, mock_record]
         got_n = provider.apply(plan)
         self.assertEquals(2, got_n)
         nsone_zone.loadRecord.assert_has_calls([
             call('unit.tests', u'A'),
-            call().update(answers=[u'1.2.3.4'], ttl=32),
             call('delete-me', u'A'),
-            call().delete()
         ])
+        mock_record.assert_has_calls([
+            call.update(answers=[u'1.2.3.4'], ttl=32),
+            call.delete()
+        ])
+
+    def test_escaping(self):
+        provider = Ns1Provider('test', 'api-key')
+
+        record = {
+            'ttl': 31,
+            'short_answers': ['foo; bar baz; blip']
+        }
+        self.assertEquals(['foo\; bar baz\; blip'],
+                          provider._data_for_SPF('SPF', record)['values'])
+
+        record = {
+            'ttl': 31,
+            'short_answers': ['no', 'foo; bar baz; blip', 'yes']
+        }
+        self.assertEquals(['no', 'foo\; bar baz\; blip', 'yes'],
+                          provider._data_for_TXT('TXT', record)['values'])
+
+        zone = Zone('unit.tests.', [])
+        record = Record.new(zone, 'spf', {
+            'ttl': 34,
+            'type': 'SPF',
+            'value': 'foo\; bar baz\; blip'
+        })
+        self.assertEquals(['foo; bar baz; blip'],
+                          provider._params_for_SPF(record)['answers'])
+
+        record = Record.new(zone, 'txt', {
+            'ttl': 35,
+            'type': 'TXT',
+            'value': 'foo\; bar baz\; blip'
+        })
+        self.assertEquals(['foo; bar baz; blip'],
+                          provider._params_for_TXT(record)['answers'])

tests/test_octodns_provider_ovh.py

@@ -0,0 +1,359 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from unittest import TestCase
+
+from mock import patch, call
+from ovh import APIError
+
+from octodns.provider.ovh import OvhProvider
+from octodns.record import Record
+from octodns.zone import Zone
+
+
+class TestOvhProvider(TestCase):
+    api_record = []
+
+    zone = Zone('unit.tests.', [])
+    expected = set()
+
+    # A, subdomain=''
+    api_record.append({
+        'fieldType': 'A',
+        'ttl': 100,
+        'target': '1.2.3.4',
+        'subDomain': '',
+        'id': 1
+    })
+    expected.add(Record.new(zone, '', {
+        'ttl': 100,
+        'type': 'A',
+        'value': '1.2.3.4',
+    }))
+
+    # A, subdomain='sub
+    api_record.append({
+        'fieldType': 'A',
+        'ttl': 200,
+        'target': '1.2.3.4',
+        'subDomain': 'sub',
+        'id': 2
+    })
+    expected.add(Record.new(zone, 'sub', {
+        'ttl': 200,
+        'type': 'A',
+        'value': '1.2.3.4',
+    }))
+
+    # CNAME
+    api_record.append({
+        'fieldType': 'CNAME',
+        'ttl': 300,
+        'target': 'unit.tests.',
+        'subDomain': 'www2',
+        'id': 3
+    })
+    expected.add(Record.new(zone, 'www2', {
+        'ttl': 300,
+        'type': 'CNAME',
+        'value': 'unit.tests.',
+    }))
+
+    # MX
+    api_record.append({
+        'fieldType': 'MX',
+        'ttl': 400,
+        'target': '10 mx1.unit.tests.',
+        'subDomain': '',
+        'id': 4
+    })
+    expected.add(Record.new(zone, '', {
+        'ttl': 400,
+        'type': 'MX',
+        'values': [{
+            'preference': 10,
+            'exchange': 'mx1.unit.tests.',
+        }]
+    }))
+
+    # NAPTR
+    api_record.append({
+        'fieldType': 'NAPTR',
+        'ttl': 500,
+        'target': '10 100 "S" "SIP+D2U" "!^.*$!sip:[email protected]!" .',
+        'subDomain': 'naptr',
+        'id': 5
+    })
+    expected.add(Record.new(zone, 'naptr', {
+        'ttl': 500,
+        'type': 'NAPTR',
+        'values': [{
+            'flags': 'S',
+            'order': 10,
+            'preference': 100,
+            'regexp': '!^.*$!sip:[email protected]!',
+            'replacement': '.',
+            'service': 'SIP+D2U',
+        }]
+    }))
+
+    # NS
+    api_record.append({
+        'fieldType': 'NS',
+        'ttl': 600,
+        'target': 'ns1.unit.tests.',
+        'subDomain': '',
+        'id': 6
+    })
+    api_record.append({
+        'fieldType': 'NS',
+        'ttl': 600,
+        'target': 'ns2.unit.tests.',
+        'subDomain': '',
+        'id': 7
+    })
+    expected.add(Record.new(zone, '', {
+        'ttl': 600,
+        'type': 'NS',
+        'values': ['ns1.unit.tests.', 'ns2.unit.tests.'],
+    }))
+
+    # NS with sub
+    api_record.append({
+        'fieldType': 'NS',
+        'ttl': 700,
+        'target': 'ns3.unit.tests.',
+        'subDomain': 'www3',
+        'id': 8
+    })
+    api_record.append({
+        'fieldType': 'NS',
+        'ttl': 700,
+        'target': 'ns4.unit.tests.',
+        'subDomain': 'www3',
+        'id': 9
+    })
+    expected.add(Record.new(zone, 'www3', {
+        'ttl': 700,
+        'type': 'NS',
+        'values': ['ns3.unit.tests.', 'ns4.unit.tests.'],
+    }))
+
+    api_record.append({
+        'fieldType': 'SRV',
+        'ttl': 800,
+        'target': '10 20 30 foo-1.unit.tests.',
+        'subDomain': '_srv._tcp',
+        'id': 10
+    })
+    api_record.append({
+        'fieldType': 'SRV',
+        'ttl': 800,
+        'target': '40 50 60 foo-2.unit.tests.',
+        'subDomain': '_srv._tcp',
+        'id': 11
+    })
+    expected.add(Record.new(zone, '_srv._tcp', {
+        'ttl': 800,
+        'type': 'SRV',
+        'values': [{
+            'priority': 10,
+            'weight': 20,
+            'port': 30,
+            'target': 'foo-1.unit.tests.',
+        }, {
+            'priority': 40,
+            'weight': 50,
+            'port': 60,
+            'target': 'foo-2.unit.tests.',
+        }]
+    }))
+
+    # PTR
+    api_record.append({
+        'fieldType': 'PTR',
+        'ttl': 900,
+        'target': 'unit.tests.',
+        'subDomain': '4',
+        'id': 12
+    })
+    expected.add(Record.new(zone, '4', {
+        'ttl': 900,
+        'type': 'PTR',
+        'value': 'unit.tests.'
+    }))
+
+    # SPF
+    api_record.append({
+        'fieldType': 'SPF',
+        'ttl': 1000,
+        'target': 'v=spf1 include:unit.texts.rerirect ~all',
+        'subDomain': '',
+        'id': 13
+    })
+    expected.add(Record.new(zone, '', {
+        'ttl': 1000,
+        'type': 'SPF',
+        'value': 'v=spf1 include:unit.texts.rerirect ~all'
+    }))
+
+    # SSHFP
+    api_record.append({
+        'fieldType': 'SSHFP',
+        'ttl': 1100,
+        'target': '1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73 ',
+        'subDomain': '',
+        'id': 14
+    })
+    expected.add(Record.new(zone, '', {
+        'ttl': 1100,
+        'type': 'SSHFP',
+        'value': {
+            'algorithm': 1,
+            'fingerprint': 'bf6b6825d2977c511a475bbefb88aad54a92ac73',
+            'fingerprint_type': 1
+        }
+    }))
+
+    # AAAA
+    api_record.append({
+        'fieldType': 'AAAA',
+        'ttl': 1200,
+        'target': '1:1ec:1::1',
+        'subDomain': '',
+        'id': 15
+    })
+    expected.add(Record.new(zone, '', {
+        'ttl': 200,
+        'type': 'AAAA',
+        'value': '1:1ec:1::1',
+    }))
+
+    @patch('ovh.Client')
+    def test_populate(self, client_mock):
+        provider = OvhProvider('test', 'endpoint', 'application_key',
+                               'application_secret', 'consumer_key')
+
+        with patch.object(provider._client, 'get') as get_mock:
+            zone = Zone('unit.tests.', [])
+            get_mock.side_effect = APIError('boom')
+            with self.assertRaises(APIError) as ctx:
+                provider.populate(zone)
+            self.assertEquals(get_mock.side_effect, ctx.exception)
+
+        with patch.object(provider._client, 'get') as get_mock:
+            zone = Zone('unit.tests.', [])
+            get_returns = [[record['id'] for record in self.api_record]]
+            get_returns += self.api_record
+            get_mock.side_effect = get_returns
+            provider.populate(zone)
+            self.assertEquals(self.expected, zone.records)
+
+    @patch('ovh.Client')
+    def test_apply(self, client_mock):
+        provider = OvhProvider('test', 'endpoint', 'application_key',
+                               'application_secret', 'consumer_key')
+
+        desired = Zone('unit.tests.', [])
+
+        for r in self.expected:
+            desired.add_record(r)
+
+        with patch.object(provider._client, 'post') as get_mock:
+            plan = provider.plan(desired)
+            get_mock.side_effect = APIError('boom')
+            with self.assertRaises(APIError) as ctx:
+                provider.apply(plan)
+            self.assertEquals(get_mock.side_effect, ctx.exception)
+
+        with patch.object(provider._client, 'get') as get_mock:
+            get_returns = [[1, 2], {
+                'fieldType': 'A',
+                'ttl': 600,
+                'target': '5.6.7.8',
+                'subDomain': '',
+                'id': 100
+            }, {'fieldType': 'A',
+                'ttl': 600,
+                'target': '5.6.7.8',
+                'subDomain': 'fake',
+                'id': 101
+                }]
+            get_mock.side_effect = get_returns
+
+            plan = provider.plan(desired)
+
+            with patch.object(provider._client, 'post') as post_mock:
+                with patch.object(provider._client, 'delete') as delete_mock:
+                    with patch.object(provider._client, 'get') as get_mock:
+                        get_mock.side_effect = [[100], [101]]
+                        provider.apply(plan)
+                        wanted_calls = [
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'A',
+                                 subDomain=u'', target=u'1.2.3.4', ttl=100),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'SRV',
+                                 subDomain=u'10 20 30 foo-1.unit.tests.',
+                                 target='_srv._tcp', ttl=800),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'SRV',
+                                 subDomain=u'40 50 60 foo-2.unit.tests.',
+                                 target='_srv._tcp', ttl=800),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'PTR', subDomain='4',
+                                 target=u'unit.tests.', ttl=900),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'NS', subDomain='www3',
+                                 target=u'ns3.unit.tests.', ttl=700),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'NS', subDomain='www3',
+                                 target=u'ns4.unit.tests.', ttl=700),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'SSHFP',
+                                 subDomain=u'1 1 bf6b6825d2977c511a475bbefb88a'
+                                           u'ad54'
+                                           u'a92ac73',
+                                 target=u'', ttl=1100),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'AAAA', subDomain=u'',
+                                 target=u'1:1ec:1::1', ttl=200),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'MX', subDomain=u'',
+                                 target=u'10 mx1.unit.tests.', ttl=400),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'CNAME', subDomain='www2',
+                                 target=u'unit.tests.', ttl=300),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'SPF', subDomain=u'',
+                                 target=u'v=spf1 include:unit.texts.'
+                                        u'rerirect ~all',
+                                 ttl=1000),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'A',
+                                 subDomain='sub', target=u'1.2.3.4', ttl=200),
+                            call(u'/domain/zone/unit.tests/record',
+                                 fieldType=u'NAPTR', subDomain='naptr',
+                                 target=u'10 100 "S" "SIP+D2U" "!^.*$!sip:'
+                                        u'info@bar'
+                                        u'.example.com!" .',
+                                 ttl=500),
+                            call(u'/domain/zone/unit.tests/refresh')]
+
+                        post_mock.assert_has_calls(wanted_calls)
+
+                        # Get for delete calls
+                        get_mock.assert_has_calls(
+                            [call(u'/domain/zone/unit.tests/record',
+                                  fieldType=u'A', subDomain=u''),
+                             call(u'/domain/zone/unit.tests/record',
+                                  fieldType=u'A', subDomain='fake')]
+                        )
+                        # 2 delete calls, one for update + one for delete
+                        delete_mock.assert_has_calls(
+                            [call(u'/domain/zone/unit.tests/record/100'),
+                             call(u'/domain/zone/unit.tests/record/101')])

tests/test_octodns_provider_powerdns.py

@@ -78,7 +78,7 @@ class TestPowerDnsProvider(TestCase):
         source = YamlProvider('test', join(dirname(__file__), 'config'))
         source.populate(expected)
         expected_n = len(expected.records) - 1
-        self.assertEquals(14, expected_n)
+        self.assertEquals(15, expected_n)
 
         # No diffs == no changes
         with requests_mock() as mock:
@@ -86,7 +86,7 @@ class TestPowerDnsProvider(TestCase):
 
             zone = Zone('unit.tests.', [])
             provider.populate(zone)
-            self.assertEquals(14, len(zone.records))
+            self.assertEquals(15, len(zone.records))
             changes = expected.changes(zone, provider)
             self.assertEquals(0, len(changes))
 
@@ -166,7 +166,7 @@ class TestPowerDnsProvider(TestCase):
         expected = Zone('unit.tests.', [])
         source = YamlProvider('test', join(dirname(__file__), 'config'))
         source.populate(expected)
-        self.assertEquals(15, len(expected.records))
+        self.assertEquals(16, len(expected.records))
 
         # A small change to a single record
         with requests_mock() as mock:
@@ -252,7 +252,7 @@ class TestPowerDnsProvider(TestCase):
             plan = provider.plan(expected)
             self.assertFalse(plan)
             # remove it now that we don't need the unrelated change any longer
-            expected.records.remove(unrelated_record)
+            expected._remove_record(unrelated_record)
 
         # ttl diff
         with requests_mock() as mock:

tests/test_octodns_provider_route53.py

@@ -58,11 +58,11 @@ class TestRoute53Provider(TestCase):
                                                       'Goodbye World?']}),
         ('', {'ttl': 64, 'type': 'MX',
               'values': [{
-                  'priority': 10,
-                  'value': 'smtp-1.unit.tests.',
+                  'preference': 10,
+                  'exchange': 'smtp-1.unit.tests.',
               }, {
-                  'priority': 20,
-                  'value': 'smtp-2.unit.tests.',
+                  'preference': 20,
+                  'exchange': 'smtp-2.unit.tests.',
               }]}),
         ('naptr', {'ttl': 65, 'type': 'NAPTR',
                    'value': {
@@ -83,6 +83,12 @@ class TestRoute53Provider(TestCase):
          {'ttl': 67, 'type': 'NS', 'values': ['8.2.3.4.', '9.2.3.4.']}),
         ('sub',
          {'ttl': 68, 'type': 'NS', 'values': ['5.2.3.4.', '6.2.3.4.']}),
+        ('',
+         {'ttl': 69, 'type': 'CAA', 'value': {
+             'flags': 0,
+             'tag': 'issue',
+             'value': 'ca.unit.tests'
+         }}),
     ):
         record = Record.new(expected, name, data)
         expected.add_record(record)
@@ -311,6 +317,13 @@ class TestRoute53Provider(TestCase):
                     'Value': 'ns1.unit.tests.',
                 }],
                 'TTL': 69,
+            }, {
+                'Name': 'unit.tests.',
+                'Type': 'CAA',
+                'ResourceRecords': [{
+                    'Value': '0 issue "ca.unit.tests"',
+                }],
+                'TTL': 69,
             }],
             'IsTruncated': False,
             'MaxItems': '100',
@@ -358,7 +371,7 @@ class TestRoute53Provider(TestCase):
                              {'HostedZoneId': 'z42'})
 
         plan = provider.plan(self.expected)
-        self.assertEquals(8, len(plan.changes))
+        self.assertEquals(9, len(plan.changes))
         for change in plan.changes:
             self.assertIsInstance(change, Create)
         stubber.assert_no_pending_responses()
@@ -377,17 +390,17 @@ class TestRoute53Provider(TestCase):
                                  'SubmittedAt': '2017-01-29T01:02:03Z',
                              }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY})
 
-        self.assertEquals(8, provider.apply(plan))
+        self.assertEquals(9, provider.apply(plan))
         stubber.assert_no_pending_responses()
 
         # Delete by monkey patching in a populate that includes an extra record
-        def add_extra_populate(existing, target):
+        def add_extra_populate(existing, target, lenient):
             for record in self.expected.records:
-                existing.records.add(record)
+                existing.add_record(record)
             record = Record.new(existing, 'extra',
                                 {'ttl': 99, 'type': 'A',
                                  'values': ['9.9.9.9']})
-            existing.records.add(record)
+            existing.add_record(record)
 
         provider.populate = add_extra_populate
         change_resource_record_sets_params = {
@@ -417,10 +430,10 @@ class TestRoute53Provider(TestCase):
 
         # Update by monkey patching in a populate that modifies the A record
         # with geos
-        def mod_geo_populate(existing, target):
+        def mod_geo_populate(existing, target, lenient):
             for record in self.expected.records:
                 if record._type != 'A' or not record.geo:
-                    existing.records.add(record)
+                    existing.add_record(record)
             record = Record.new(existing, '', {
                 'ttl': 61,
                 'type': 'A',
@@ -431,7 +444,7 @@ class TestRoute53Provider(TestCase):
                     'NA-US-KY': ['7.2.3.4']
                 }
             })
-            existing.records.add(record)
+            existing.add_record(record)
 
         provider.populate = mod_geo_populate
         change_resource_record_sets_params = {
@@ -513,10 +526,10 @@ class TestRoute53Provider(TestCase):
 
         # Update converting to non-geo by monkey patching in a populate that
         # modifies the A record with geos
-        def mod_add_geo_populate(existing, target):
+        def mod_add_geo_populate(existing, target, lenient):
             for record in self.expected.records:
                 if record._type != 'A' or record.geo:
-                    existing.records.add(record)
+                    existing.add_record(record)
             record = Record.new(existing, 'simple', {
                 'ttl': 61,
                 'type': 'A',
@@ -525,7 +538,7 @@ class TestRoute53Provider(TestCase):
                     'OC': ['3.2.3.4', '4.2.3.4'],
                 }
             })
-            existing.records.add(record)
+            existing.add_record(record)
 
         provider.populate = mod_add_geo_populate
         change_resource_record_sets_params = {
@@ -590,7 +603,7 @@ class TestRoute53Provider(TestCase):
                              {})
 
         plan = provider.plan(self.expected)
-        self.assertEquals(8, len(plan.changes))
+        self.assertEquals(9, len(plan.changes))
         for change in plan.changes:
             self.assertIsInstance(change, Create)
         stubber.assert_no_pending_responses()
@@ -637,7 +650,7 @@ class TestRoute53Provider(TestCase):
                                  'SubmittedAt': '2017-01-29T01:02:03Z',
                              }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY})
 
-        self.assertEquals(8, provider.apply(plan))
+        self.assertEquals(9, provider.apply(plan))
         stubber.assert_no_pending_responses()
 
     def test_health_checks_pagination(self):
@@ -1269,16 +1282,16 @@ class TestRoute53Provider(TestCase):
     @patch('octodns.provider.route53.Route53Provider._really_apply')
     def test_apply_1(self, really_apply_mock):
 
-        # 17 RRs with max of 18 should only get applied in one call
-        provider, plan = self._get_test_plan(18)
+        # 18 RRs with max of 19 should only get applied in one call
+        provider, plan = self._get_test_plan(19)
         provider.apply(plan)
         really_apply_mock.assert_called_once()
 
     @patch('octodns.provider.route53.Route53Provider._really_apply')
     def test_apply_2(self, really_apply_mock):
 
-        # 17 RRs with max of 17 should only get applied in two calls
-        provider, plan = self._get_test_plan(17)
+        # 18 RRs with max of 17 should only get applied in two calls
+        provider, plan = self._get_test_plan(18)
         provider.apply(plan)
         self.assertEquals(2, really_apply_mock.call_count)
 
@@ -1327,6 +1340,14 @@ class TestRoute53Provider(TestCase):
             'Type': 'TXT',
         }))
 
+    def test_client_max_attempts(self):
+        provider = Route53Provider('test', 'abc', '123',
+                                   client_max_attempts=42)
+        # NOTE: this will break if boto ever changes the impl details...
+        self.assertEquals(43, provider._conn.meta.events
+                          ._unique_id_handlers['retry-config-route53']
+                          ['handler']._checker.__dict__['_max_attempts'])
+
 
 class TestRoute53Records(TestCase):
 
@@ -1355,8 +1376,10 @@ class TestRoute53Records(TestCase):
                            False)
         self.assertEquals(c, c)
         d = _Route53Record(None, Record.new(existing, '',
-                                            {'ttl': 42, 'type': 'CNAME',
-                                             'value': 'foo.bar.'}),
+                                            {'ttl': 42, 'type': 'MX',
+                                             'value': {
+                                                 'preference': 10,
+                                                 'exchange': 'foo.bar.'}}),
                            False)
         self.assertEquals(d, d)
 

tests/test_octodns_provider_yaml.py

@@ -30,7 +30,7 @@ class TestYamlProvider(TestCase):
 
         # without it we see everything
         source.populate(zone)
-        self.assertEquals(15, len(zone.records))
+        self.assertEquals(16, len(zone.records))
 
         # Assumption here is that a clean round-trip means that everything
         # worked as expected, data that went in came back out and could be

tests/test_octodns_source_tinydns.py

@@ -68,22 +68,22 @@ class TestTinyDnsFileSource(TestCase):
                 'type': 'MX',
                 'ttl': 3600,
                 'values': [{
-                    'priority': 10,
-                    'value': 'smtp-1-host.example.com.',
+                    'preference': 10,
+                    'exchange': 'smtp-1-host.example.com.',
                 }, {
-                    'priority': 20,
-                    'value': 'smtp-2-host.example.com.',
+                    'preference': 20,
+                    'exchange': 'smtp-2-host.example.com.',
                 }]
             }),
             ('smtp', {
                 'type': 'MX',
                 'ttl': 1800,
                 'values': [{
-                    'priority': 30,
-                    'value': 'smtp-1-host.example.com.',
+                    'preference': 30,
+                    'exchange': 'smtp-1-host.example.com.',
                 }, {
-                    'priority': 40,
-                    'value': 'smtp-2-host.example.com.',
+                    'preference': 40,
+                    'exchange': 'smtp-2-host.example.com.',
                 }]
             }),
         ):

tests/test_octodns_yaml.py

@@ -48,8 +48,8 @@ class TestYaml(TestCase):
 '*.11.2': 'd'
 '*.10.1': 'c'
 ''')
-        self.assertEquals('keys out of order: *.2.2, *.1.2, *.11.2, *.10.1',
-                          ctx.exception.problem)
+        self.assertTrue('keys out of order: expected *.1.2 got *.2.2 at' in
+                        ctx.exception.problem)
 
         buf = StringIO()
         safe_dump({

tests/test_octodns_zone.py

@@ -8,7 +8,8 @@ from __future__ import absolute_import, division, print_function, \
 from unittest import TestCase
 
 from octodns.record import ARecord, AaaaRecord, Create, Delete, Record, Update
-from octodns.zone import DuplicateRecordException, SubzoneRecordException, Zone
+from octodns.zone import DuplicateRecordException, InvalidNodeException, \
+    SubzoneRecordException, Zone
 
 from helpers import SimpleProvider
 
@@ -38,6 +39,7 @@ class TestZone(TestCase):
 
         a = ARecord(zone, 'a', {'ttl': 42, 'value': '1.1.1.1'})
         b = ARecord(zone, 'b', {'ttl': 42, 'value': '1.1.1.1'})
+        c = ARecord(zone, 'a', {'ttl': 43, 'value': '2.2.2.2'})
 
         zone.add_record(a)
         self.assertEquals(zone.records, set([a]))
@@ -47,6 +49,11 @@ class TestZone(TestCase):
         self.assertEquals('Duplicate record a.unit.tests., type A',
                           ctx.exception.message)
         self.assertEquals(zone.records, set([a]))
+
+        # can add duplicate with replace=True
+        zone.add_record(c, replace=True)
+        self.assertEquals('2.2.2.2', list(zone.records)[0].values[0])
+
         # Can add dup name, with different type
         zone.add_record(b)
         self.assertEquals(zone.records, set([a, b]))
@@ -70,7 +77,7 @@ class TestZone(TestCase):
         # add a record, delete a record -> [Delete, Create]
         c = ARecord(before, 'c', {'ttl': 42, 'value': '1.1.1.1'})
         after.add_record(c)
-        after.records.remove(b)
+        after._remove_record(b)
         self.assertEquals(after.records, set([a, c]))
         changes = before.changes(after, target)
         self.assertEquals(2, len(changes))
@@ -205,3 +212,27 @@ class TestZone(TestCase):
 
         self.assertTrue(zone_missing.changes(zone_normal, provider))
         self.assertFalse(zone_missing.changes(zone_ignored, provider))
+
+    def test_cname_coexisting(self):
+        zone = Zone('unit.tests.', [])
+        a = Record.new(zone, 'www', {
+            'ttl': 60,
+            'type': 'A',
+            'value': '9.9.9.9',
+        })
+        cname = Record.new(zone, 'www', {
+            'ttl': 60,
+            'type': 'CNAME',
+            'value': 'foo.bar.com.',
+        })
+
+        # add cname to a
+        zone.add_record(a)
+        with self.assertRaises(InvalidNodeException):
+            zone.add_record(cname)
+
+        # add a to cname
+        zone = Zone('unit.tests.', [])
+        zone.add_record(cname)
+        with self.assertRaises(InvalidNodeException):
+            zone.add_record(a)