Merge branch 'master' into add_rackspace_provider

.gitignore

@@ -9,3 +9,5 @@ nosetests.xml
 octodns.egg-info/
 output/
 tmp/
+build/
+config/

CHANGELOG.md

@@ -0,0 +1,55 @@
+## v0.8.5 - 2017-07-21 - Azure, NS1 escaping, & large zones
+
+Relatively small delta this go around. No major themes or anything, just steady
+progress.
+
+* AzureProvider added thanks to work by
+  [Heesu Hwang](https://github.com/h-hwang).
+* Fixed some escaping issues with NS1 TXT and SPF records that were tracked down
+  with the help of [Blake Stoddard](https://github.com/blakestoddard).
+* Some tweaks were made to Zone.records to vastly improve handling of zones with
+  very large numbers of records, no more O(N^2).
+
+## v0.8.4 - 2017-06-28 - It's been too long
+
+Lots of updates based on our internal use, needs, and feedback & suggestions
+from our OSS users. There's too much to list out since the previous release was
+cut, but I'll try to cover the highlights/important bits and promise to do
+better in the future :fingers_crossed:
+
+#### Major:
+
+* Complete rework of record validation with lenient mode support added to
+  octodns-dump so that data with validation problems can be dumped to config
+  files as a starting point. octoDNS now also ignores validation errors when
+  pulling the current state from a provider before planning changes. In both
+  cases this is best effort.
+* Naming of record keys are based on RFC-1035 and friends, previous names have
+  been kept for backwards compatibility until the 1.0 release.
+* Provider record type support is now explicit, i.e. opt-in, rather than
+  opt-out. This prevents bugs/oversights in record handling where providers
+  don't support (new) record types and didn't correctly ignore them.
+* ALIAS support for DNSimple, Dyn, NS1, PowerDNS
+* Ignored record support added, `octodns:\n  ignored: True`
+* Ns1Provider added
+
+#### Miscellaneous
+
+* Use a 3rd party lib for nautrual sorting of keys, rather than my old
+  implementation. Sorting can be disabled in the YamlProvider with
+  `enforce_order: False`.
+* Semi-colon/escaping fixes and improvements.
+* Meta record support, `TXT octodns-meta.<zone>`. For now just
+  `provider=<provider-id>`. Optionally turned on with `include_meta` manager
+  config val.
+* Validations check for CNAMEs co-existing with other records and error out if
+  found. Was a common mistaken/unknown issue and this surfaces the problem
+  early.
+* Sizeable refactor in the way Route53 record translation works to make it
+  cleaner/less hacky
+* Lots of docs type-o fixes
+* Fixed some pretty major bugs in DnsimpleProvider
+* Relax UnsafePlan checks a bit, more to come here
+* Set User-Agent header on Dyn health checks
+
+## v0.8.0 - 2017-03-14 - First public release

README.md

@@ -149,12 +149,13 @@ The above command pulled the existing data out of Route53 and placed the results
 
 | Provider | Record Support | GeoDNS Support | Notes |
 |--|--|--|--|
-| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CNAME, MX, NS, SPF, TXT | No | |
-| [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | |
+| [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | |
+| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted |
+| [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted |
 | [DynProvider](/octodns/provider/dyn.py) | All | Yes | |
 | [Ns1Provider](/octodns/provider/ns1.py) | All | No | |
 | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | |
-| [Route53](/octodns/provider/route53.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | |
+| [Route53](/octodns/provider/route53.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | |
 | [TinyDNSSource](/octodns/source/tinydns.py) | A, CNAME, MX, NS, PTR | No | read-only |
 | [YamlProvider](/octodns/provider/yaml.py) | All | Yes | config |
 

octodns/__init__.py

@@ -1,8 +1,6 @@
-'''
-OctoDNS: DNS as code - Tools for managing DNS across multiple providers
-'''
+'OctoDNS: DNS as code - Tools for managing DNS across multiple providers'
 
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
-__VERSION__ = '0.8.0'
+__VERSION__ = '0.8.6'

octodns/cmds/dump.py

@@ -18,6 +18,9 @@ def main():
     parser.add_argument('--output-dir', required=True,
                         help='The directory into which the results will be '
                         'written (Note: will overwrite existing files)')
+    parser.add_argument('--lenient', action='store_true', default=False,
+                        help='Ignore record validations and do a best effort '
+                        'dump')
     parser.add_argument('zone', help='Zone to dump')
     parser.add_argument('source', nargs='+',
                         help='Source(s) to pull data from')
@@ -25,7 +28,7 @@ def main():
     args = parser.parse_args()
 
     manager = Manager(args.config_file)
-    manager.dump(args.zone, args.output_dir, *args.source)
+    manager.dump(args.zone, args.output_dir, args.lenient, *args.source)
 
 
 if __name__ == '__main__':

octodns/manager.py

@@ -6,13 +6,14 @@ from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
 from StringIO import StringIO
-from concurrent.futures import Future, ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor
 from importlib import import_module
 from os import environ
 import logging
 
 from .provider.base import BaseProvider
 from .provider.yaml import YamlProvider
+from .record import Record
 from .yaml import safe_load
 from .zone import Zone
 
@@ -37,6 +38,17 @@ class _AggregateTarget(object):
         return True
 
 
+class MakeThreadFuture(object):
+
+    def __init__(self, func, args, kwargs):
+        self.func = func
+        self.args = args
+        self.kwargs = kwargs
+
+    def result(self):
+        return self.func(*self.args, **self.kwargs)
+
+
 class MainThreadExecutor(object):
     '''
     Dummy executor that runs things on the main thread during the involcation
@@ -47,19 +59,13 @@ class MainThreadExecutor(object):
     '''
 
     def submit(self, func, *args, **kwargs):
-        future = Future()
-        try:
-            future.set_result(func(*args, **kwargs))
-        except Exception as e:
-            # TODO: get right stacktrace here
-            future.set_exception(e)
-        return future
+        return MakeThreadFuture(func, args, kwargs)
 
 
 class Manager(object):
     log = logging.getLogger('Manager')
 
-    def __init__(self, config_file, max_workers=None):
+    def __init__(self, config_file, max_workers=None, include_meta=False):
         self.log.info('__init__: config_file=%s', config_file)
 
         # Read our config file
@@ -69,11 +75,16 @@ class Manager(object):
         manager_config = self.config.get('manager', {})
         max_workers = manager_config.get('max_workers', 1) \
             if max_workers is None else max_workers
+        self.log.info('__init__:   max_workers=%d', max_workers)
         if max_workers > 1:
             self._executor = ThreadPoolExecutor(max_workers=max_workers)
         else:
             self._executor = MainThreadExecutor()
 
+        self.include_meta = include_meta or manager_config.get('include_meta',
+                                                               False)
+        self.log.info('__init__:   max_workers=%s', self.include_meta)
+
         self.log.debug('__init__:   configuring providers')
         self.providers = {}
         for provider_name, provider_config in self.config['providers'].items():
@@ -175,6 +186,13 @@ class Manager(object):
         plans = []
 
         for target in targets:
+            if self.include_meta:
+                meta = Record.new(zone, 'octodns-meta', {
+                    'type': 'TXT',
+                    'ttl': 60,
+                    'value': 'provider={}'.format(target.id)
+                })
+                zone.add_record(meta, replace=True)
             plan = target.plan(zone)
             if plan:
                 plans.append((target, plan))
@@ -322,7 +340,7 @@ class Manager(object):
 
         return zb.changes(za, _AggregateTarget(a + b))
 
-    def dump(self, zone, output_dir, source, *sources):
+    def dump(self, zone, output_dir, lenient, source, *sources):
         '''
         Dump zone data from the specified source
         '''
@@ -341,7 +359,7 @@ class Manager(object):
 
         zone = Zone(zone, self.configured_sub_zones(zone))
         for source in sources:
-            source.populate(zone)
+            source.populate(zone, lenient=lenient)
 
         plan = target.plan(zone)
         target.apply(plan)

octodns/provider/azuredns.py

@@ -0,0 +1,443 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from azure.common.credentials import ServicePrincipalCredentials
+from azure.mgmt.dns import DnsManagementClient
+from msrestazure.azure_exceptions import CloudError
+
+from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \
+    SrvRecord, NsRecord, PtrRecord, TxtRecord, Zone
+
+import logging
+from functools import reduce
+from ..record import Record
+from .base import BaseProvider
+
+
+class _AzureRecord(object):
+    '''Wrapper for OctoDNS record for AzureProvider to make dns_client calls.
+
+        azuredns.py:
+        class: octodns.provider.azuredns._AzureRecord
+        An _AzureRecord is easily accessible to Azure DNS Management library
+        functions and is used to wrap all relevant data to create a record in
+        Azure.
+    '''
+    TYPE_MAP = {
+        'A': ARecord,
+        'AAAA': AaaaRecord,
+        'CNAME': CnameRecord,
+        'MX': MxRecord,
+        'SRV': SrvRecord,
+        'NS': NsRecord,
+        'PTR': PtrRecord,
+        'TXT': TxtRecord
+    }
+
+    def __init__(self, resource_group, record, delete=False):
+        '''Contructor for _AzureRecord.
+
+            Notes on Azure records: An Azure record set has the form
+            RecordSet(name=<...>, type=<...>, arecords=[...], aaaa_records, ..)
+            When constructing an azure record as done in self._apply_Create,
+            the argument parameters for an A record would be
+            parameters={'ttl': <int>, 'arecords': [ARecord(<str ip>),]}.
+            As another example for CNAME record:
+            parameters={'ttl': <int>, 'cname_record': CnameRecord(<str>)}.
+
+            Below, key_name and class_name are the dictionary key and Azure
+            Record class respectively.
+
+            :param resource_group: The name of resource group in Azure
+            :type  resource_group: str
+            :param record: An OctoDNS record
+            :type  record: ..record.Record
+            :param delete: If true, omit data parsing; not needed to delete
+            :type  delete: bool
+
+            :type return: _AzureRecord
+        '''
+        self.resource_group = resource_group
+        self.zone_name = record.zone.name[:len(record.zone.name) - 1]
+        self.relative_record_set_name = record.name or '@'
+        self.record_type = record._type
+
+        if delete:
+            return
+
+        # Refer to function docstring for key_name and class_name.
+        format_u_s = '' if record._type == 'A' else '_'
+        key_name = '{}{}records'.format(self.record_type, format_u_s).lower()
+        if record._type == 'CNAME':
+            key_name = key_name[:len(key_name) - 1]
+        azure_class = self.TYPE_MAP[self.record_type]
+
+        self.params = getattr(self, '_params_for_{}'.format(record._type))
+        self.params = self.params(record.data, key_name, azure_class)
+        self.params['ttl'] = record.ttl
+
+    def _params(self, data, key_name, azure_class):
+        try:
+            values = data['values']
+        except KeyError:
+            values = [data['value']]
+        return {key_name: [azure_class(v) for v in values]}
+
+    _params_for_A = _params
+    _params_for_AAAA = _params
+    _params_for_NS = _params
+    _params_for_PTR = _params
+
+    def _params_for_CNAME(self, data, key_name, azure_class):
+        return {key_name: azure_class(data['value'])}
+
+    def _params_for_MX(self, data, key_name, azure_class):
+        params = []
+        if 'values' in data:
+            for vals in data['values']:
+                params.append(azure_class(vals['preference'],
+                                          vals['exchange']))
+        else:  # Else there is a singular data point keyed by 'value'.
+            params.append(azure_class(data['value']['preference'],
+                                      data['value']['exchange']))
+        return {key_name: params}
+
+    def _params_for_SRV(self, data, key_name, azure_class):
+        params = []
+        if 'values' in data:
+            for vals in data['values']:
+                params.append(azure_class(vals['priority'],
+                                          vals['weight'],
+                                          vals['port'],
+                                          vals['target']))
+        else:  # Else there is a singular data point keyed by 'value'.
+            params.append(azure_class(data['value']['priority'],
+                                      data['value']['weight'],
+                                      data['value']['port'],
+                                      data['value']['target']))
+        return {key_name: params}
+
+    def _params_for_TXT(self, data, key_name, azure_class):
+        try:  # API for TxtRecord has list of str, even for singleton
+            values = data['values']
+        except KeyError:
+            values = [data['value']]
+        return {key_name: [azure_class([v]) for v in values]}
+
+    def _equals(self, b):
+        '''Checks whether two records are equal by comparing all fields.
+            :param b: Another _AzureRecord object
+            :type  b: _AzureRecord
+
+            :type return: bool
+        '''
+        def parse_dict(params):
+            vals = []
+            for char in params:
+                if char != 'ttl':
+                    list_records = params[char]
+                    try:
+                        for record in list_records:
+                            vals.append(record.__dict__)
+                    except:
+                        vals.append(list_records.__dict__)
+            vals.sort()
+            return vals
+
+        return (self.resource_group == b.resource_group) & \
+               (self.zone_name == b.zone_name) & \
+               (self.record_type == b.record_type) & \
+               (self.params['ttl'] == b.params['ttl']) & \
+               (parse_dict(self.params) == parse_dict(b.params)) & \
+               (self.relative_record_set_name == b.relative_record_set_name)
+
+    def __str__(self):
+        '''String representation of an _AzureRecord.
+            :type return: str
+        '''
+        string = 'Zone: {}; '.format(self.zone_name)
+        string += 'Name: {}; '.format(self.relative_record_set_name)
+        string += 'Type: {}; '.format(self.record_type)
+        if not hasattr(self, 'params'):
+            return string
+        string += 'Ttl: {}; '.format(self.params['ttl'])
+        for char in self.params:
+            if char != 'ttl':
+                try:
+                    for rec in self.params[char]:
+                        string += 'Record: {}; '.format(rec.__dict__)
+                except:
+                    string += 'Record: {}; '.format(self.params[char].__dict__)
+        return string
+
+
+def _check_endswith_dot(string):
+    return string if string.endswith('.') else string + '.'
+
+
+def _parse_azure_type(string):
+    '''Converts string representing an Azure RecordSet type to usual type.
+
+        :param string: the Azure type. eg: <Microsoft.Network/dnszones/A>
+        :type  string: str
+
+        :type return: str
+    '''
+    return string.split('/')[len(string.split('/')) - 1]
+
+
+class AzureProvider(BaseProvider):
+    '''
+    Azure DNS Provider
+
+    azuredns.py:
+        class: octodns.provider.azuredns.AzureProvider
+        # Current support of authentication of access to Azure services only
+        # includes using a Service Principal:
+        # https://docs.microsoft.com/en-us/azure/azure-resource-manager/
+        #                        resource-group-create-service-principal-portal
+        # The Azure Active Directory Application ID (aka client ID):
+        client_id:
+        # Authentication Key Value: (note this should be secret)
+        key:
+        # Directory ID (aka tenant ID):
+        directory_id:
+        # Subscription ID:
+        sub_id:
+        # Resource Group name:
+        resource_group:
+        # All are required to authenticate.
+
+        Example config file with variables:
+            "
+            ---
+            providers:
+              config:
+                class: octodns.provider.yaml.YamlProvider
+                directory: ./config (example path to directory of zone files)
+              azuredns:
+                class: octodns.provider.azuredns.AzureProvider
+                client_id: env/AZURE_APPLICATION_ID
+                key: env/AZURE_AUTHENICATION_KEY
+                directory_id: env/AZURE_DIRECTORY_ID
+                sub_id: env/AZURE_SUBSCRIPTION_ID
+                resource_group: 'TestResource1'
+
+            zones:
+              example.com.:
+                sources:
+                  - config
+                targets:
+                  - azuredns
+            "
+        The first four variables above can be hidden in environment variables
+        and octoDNS will automatically search for them in the shell. It is
+        possible to also hard-code into the config file: eg, resource_group.
+    '''
+    SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'PTR', 'SRV', 'TXT'))
+
+    def __init__(self, id, client_id, key, directory_id, sub_id,
+                 resource_group, *args, **kwargs):
+        self.log = logging.getLogger('AzureProvider[{}]'.format(id))
+        self.log.debug('__init__: id=%s, client_id=%s, '
+                       'key=***, directory_id:%s', id, client_id, directory_id)
+        super(AzureProvider, self).__init__(id, *args, **kwargs)
+
+        credentials = ServicePrincipalCredentials(
+            client_id, secret=key, tenant=directory_id
+        )
+        self._dns_client = DnsManagementClient(credentials, sub_id)
+        self._resource_group = resource_group
+        self._azure_zones = set()
+
+    def _populate_zones(self):
+        self.log.debug('azure_zones: loading')
+        list_zones = self._dns_client.zones.list_by_resource_group
+        for zone in list_zones(self._resource_group):
+            self._azure_zones.add(zone.name)
+
+    def _check_zone(self, name, create=False):
+        '''Checks whether a zone specified in a source exist in Azure server.
+
+            Note that Azure zones omit end '.' eg: contoso.com vs contoso.com.
+            Returns the name if it exists.
+
+            :param name: Name of a zone to checks
+            :type  name: str
+            :param create: If True, creates the zone of that name.
+            :type  create: bool
+
+            :type return: str or None
+        '''
+        self.log.debug('_check_zone: name=%s', name)
+        try:
+            if name in self._azure_zones:
+                return name
+            self._dns_client.zones.get(self._resource_group, name)
+            self._azure_zones.add(name)
+            return name
+        except CloudError as err:
+            msg = 'The Resource \'Microsoft.Network/dnszones/{}\''.format(name)
+            msg += ' under resource group \'{}\''.format(self._resource_group)
+            msg += ' was not found.'
+            if msg == err.message:
+                # Then the only error is that the zone doesn't currently exist
+                if create:
+                    self.log.debug('_check_zone:no matching zone; creating %s',
+                                   name)
+                    create_zone = self._dns_client.zones.create_or_update
+                    create_zone(self._resource_group, name, Zone('global'))
+                    return name
+                else:
+                    return
+            raise
+
+    def populate(self, zone, target=False, lenient=False):
+        '''Required function of manager.py to collect records from zone.
+
+            Special notes for Azure.
+            Azure zone names omit final '.'
+            Azure root records names are represented by '@'. OctoDNS uses ''
+            Azure records created through online interface may have null values
+            (eg, no IP address for A record).
+            Azure online interface allows constructing records with null values
+            which are destroyed by _apply.
+
+            Specific quirks such as these are responsible for any non-obvious
+            parsing in this function and the functions '_params_for_*'.
+
+            :param zone: A dns zone
+            :type  zone: octodns.zone.Zone
+            :param target: Checks if Azure is source or target of config.
+                           Currently only supports as a target. Unused.
+            :type  target: bool
+            :param lenient: Unused. Check octodns.manager for usage.
+            :type  lenient: bool
+
+            :type return: void
+        '''
+        self.log.debug('populate: name=%s', zone.name)
+        before = len(zone.records)
+
+        zone_name = zone.name[:len(zone.name) - 1]
+        self._populate_zones()
+        self._check_zone(zone_name)
+
+        _records = set()
+        records = self._dns_client.record_sets.list_by_dns_zone
+        if self._check_zone(zone_name):
+            for azrecord in records(self._resource_group, zone_name):
+                if _parse_azure_type(azrecord.type) in self.SUPPORTS:
+                    _records.add(azrecord)
+            for azrecord in _records:
+                record_name = azrecord.name if azrecord.name != '@' else ''
+                typ = _parse_azure_type(azrecord.type)
+                data = getattr(self, '_data_for_{}'.format(typ))
+                data = data(azrecord)
+                data['type'] = typ
+                data['ttl'] = azrecord.ttl
+                record = Record.new(zone, record_name, data, source=self)
+                zone.add_record(record)
+
+        self.log.info('populate: found %s records', len(zone.records) - before)
+
+    def _data_for_A(self, azrecord):
+        return {'values': [ar.ipv4_address for ar in azrecord.arecords]}
+
+    def _data_for_AAAA(self, azrecord):
+        return {'values': [ar.ipv6_address for ar in azrecord.aaaa_records]}
+
+    def _data_for_CNAME(self, azrecord):
+        '''Parsing data from Azure DNS Client record call
+            :param azrecord: a return of a call to list azure records
+            :type  azrecord: azure.mgmt.dns.models.RecordSet
+
+            :type  return: dict
+
+            CNAME and PTR both use the catch block to catch possible empty
+            records. Refer to population comment.
+        '''
+        try:
+            return {'value': _check_endswith_dot(azrecord.cname_record.cname)}
+        except:
+            return {'value': '.'}
+
+    def _data_for_MX(self, azrecord):
+        return {'values': [{'preference': ar.preference,
+                            'exchange': ar.exchange}
+                           for ar in azrecord.mx_records]}
+
+    def _data_for_NS(self, azrecord):
+        vals = [ar.nsdname for ar in azrecord.ns_records]
+        return {'values': [_check_endswith_dot(val) for val in vals]}
+
+    def _data_for_PTR(self, azrecord):
+        try:
+            ptrdname = azrecord.ptr_records[0].ptrdname
+            return {'value': _check_endswith_dot(ptrdname)}
+        except:
+            return {'value': '.'}
+
+    def _data_for_SRV(self, azrecord):
+        return {'values': [{'priority': ar.priority, 'weight': ar.weight,
+                            'port': ar.port, 'target': ar.target}
+                           for ar in azrecord.srv_records]}
+
+    def _data_for_TXT(self, azrecord):
+        return {'values': [reduce((lambda a, b: a + b), ar.value)
+                           for ar in azrecord.txt_records]}
+
+    def _apply_Create(self, change):
+        '''A record from change must be created.
+
+            :param change: a change object
+            :type  change: octodns.record.Change
+
+            :type return: void
+        '''
+        ar = _AzureRecord(self._resource_group, change.new)
+        create = self._dns_client.record_sets.create_or_update
+
+        create(resource_group_name=ar.resource_group,
+               zone_name=ar.zone_name,
+               relative_record_set_name=ar.relative_record_set_name,
+               record_type=ar.record_type,
+               parameters=ar.params)
+
+        self.log.debug('*  Success Create/Update: {}'.format(ar))
+
+    _apply_Update = _apply_Create
+
+    def _apply_Delete(self, change):
+        ar = _AzureRecord(self._resource_group, change.existing, delete=True)
+        delete = self._dns_client.record_sets.delete
+
+        delete(self._resource_group, ar.zone_name, ar.relative_record_set_name,
+               ar.record_type)
+
+        self.log.debug('*  Success Delete: {}'.format(ar))
+
+    def _apply(self, plan):
+        '''Required function of manager.py to actually apply a record change.
+
+            :param plan: Contains the zones and changes to be made
+            :type  plan: octodns.provider.base.Plan
+
+            :type return: void
+        '''
+        desired = plan.desired
+        changes = plan.changes
+        self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name,
+                       len(changes))
+
+        azure_zone_name = desired.name[:len(desired.name) - 1]
+        self._check_zone(azure_zone_name, create=True)
+
+        for change in changes:
+            class_name = change.__class__.__name__
+            getattr(self, '_apply_{}'.format(class_name))(change)

octodns/provider/base.py

@@ -56,19 +56,19 @@ class Plan(object):
             delete_pcent = self.change_counts['Delete'] / existing_record_count
 
             if update_pcent > self.MAX_SAFE_UPDATE_PCENT:
-                raise UnsafePlan('Too many updates, %s is over %s percent'
-                                 '(%s/%s)',
-                                 update_pcent,
-                                 self.MAX_SAFE_UPDATE_PCENT * 100,
-                                 self.change_counts['Update'],
-                                 existing_record_count)
+                raise UnsafePlan('Too many updates, {} is over {} percent'
+                                 '({}/{})'.format(
+                                     update_pcent,
+                                     self.MAX_SAFE_UPDATE_PCENT * 100,
+                                     self.change_counts['Update'],
+                                     existing_record_count))
             if delete_pcent > self.MAX_SAFE_DELETE_PCENT:
-                raise UnsafePlan('Too many deletes, %s is over %s percent'
-                                 '(%s/%s)',
-                                 delete_pcent,
-                                 self.MAX_SAFE_DELETE_PCENT * 100,
-                                 self.change_counts['Delete'],
-                                 existing_record_count)
+                raise UnsafePlan('Too many deletes, {} is over {} percent'
+                                 '({}/{})'.format(
+                                     delete_pcent,
+                                     self.MAX_SAFE_DELETE_PCENT * 100,
+                                     self.change_counts['Delete'],
+                                     existing_record_count))
 
     def __repr__(self):
         return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \
@@ -104,7 +104,7 @@ class BaseProvider(BaseSource):
         self.log.info('plan: desired=%s', desired.name)
 
         existing = Zone(desired.name, desired.sub_zones)
-        self.populate(existing, target=True)
+        self.populate(existing, target=True, lenient=True)
 
         # compute the changes at the zone/record level
         changes = existing.changes(desired, self)

octodns/provider/cloudflare.py

@@ -36,7 +36,7 @@ class CloudflareProvider(BaseProvider):
     '''
     SUPPORTS_GEO = False
     # TODO: support SRV
-    UNSUPPORTED_TYPES = ('ALIAS', 'NAPTR', 'PTR', 'SOA', 'SRV', 'SSHFP')
+    SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SPF', 'TXT'))
 
     MIN_TTL = 120
     TIMEOUT = 15
@@ -56,9 +56,6 @@ class CloudflareProvider(BaseProvider):
         self._zones = None
         self._zone_records = {}
 
-    def supports(self, record):
-        return record._type not in self.UNSUPPORTED_TYPES
-
     def _request(self, method, path, params=None, data=None):
         self.log.debug('_request: method=%s, path=%s', method, path)
 
@@ -107,6 +104,17 @@ class CloudflareProvider(BaseProvider):
             'values': [r['content'].replace(';', '\;') for r in records],
         }
 
+    def _data_for_CAA(self, _type, records):
+        values = []
+        for r in records:
+            data = r['data']
+            values.append(data)
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values,
+        }
+
     def _data_for_CNAME(self, _type, records):
         only = records[0]
         return {
@@ -119,8 +127,8 @@ class CloudflareProvider(BaseProvider):
         values = []
         for r in records:
             values.append({
-                'priority': r['priority'],
-                'value': '{}.'.format(r['content']),
+                'preference': r['priority'],
+                'exchange': '{}.'.format(r['content']),
             })
         return {
             'ttl': records[0]['ttl'],
@@ -157,8 +165,9 @@ class CloudflareProvider(BaseProvider):
 
         return self._zone_records[zone.name]
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: name=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
 
         before = len(zone.records)
         records = self.zone_records(zone)
@@ -167,14 +176,15 @@ class CloudflareProvider(BaseProvider):
             for record in records:
                 name = zone.hostname_from_fqdn(record['name'])
                 _type = record['type']
-                if _type not in self.UNSUPPORTED_TYPES:
+                if _type in self.SUPPORTS:
                     values[name][record['type']].append(record)
 
             for name, types in values.items():
                 for _type, records in types.items():
                     data_for = getattr(self, '_data_for_{}'.format(_type))
                     data = data_for(_type, records)
-                    record = Record.new(zone, name, data, source=self)
+                    record = Record.new(zone, name, data, source=self,
+                                        lenient=lenient)
                     zone.add_record(record)
 
         self.log.info('populate:   found %s records',
@@ -198,6 +208,16 @@ class CloudflareProvider(BaseProvider):
     _contents_for_NS = _contents_for_multiple
     _contents_for_SPF = _contents_for_multiple
 
+    def _contents_for_CAA(self, record):
+        for value in record.values:
+            yield {
+                'data': {
+                    'flags': value.flags,
+                    'tag': value.tag,
+                    'value': value.value,
+                }
+            }
+
     def _contents_for_TXT(self, record):
         for value in record.values:
             yield {'content': value.replace('\;', ';')}
@@ -208,8 +228,8 @@ class CloudflareProvider(BaseProvider):
     def _contents_for_MX(self, record):
         for value in record.values:
             yield {
-                'priority': value.priority,
-                'content': value.value
+                'priority': value.preference,
+                'content': value.exchange
             }
 
     def _apply_Create(self, change):

octodns/provider/dnsimple.py

@@ -91,6 +91,8 @@ class DnsimpleProvider(BaseProvider):
         account: 42
     '''
     SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS',
+                    'PTR', 'SPF', 'SRV', 'SSHFP', 'TXT'))
 
     def __init__(self, id, token, account, *args, **kwargs):
         self.log = logging.getLogger('DnsimpleProvider[{}]'.format(id))
@@ -112,6 +114,21 @@ class DnsimpleProvider(BaseProvider):
     _data_for_SPF = _data_for_multiple
     _data_for_TXT = _data_for_multiple
 
+    def _data_for_CAA(self, _type, records):
+        values = []
+        for record in records:
+            flags, tag, value = record['content'].split(' ')
+            values.append({
+                'flags': flags,
+                'tag': tag,
+                'value': value[1:-1],
+            })
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values
+        }
+
     def _data_for_CNAME(self, _type, records):
         record = records[0]
         return {
@@ -126,8 +143,8 @@ class DnsimpleProvider(BaseProvider):
         values = []
         for record in records:
             values.append({
-                'priority': record['priority'],
-                'value': '{}.'.format(record['content'])
+                'preference': record['priority'],
+                'exchange': '{}.'.format(record['content'])
             })
         return {
             'ttl': records[0]['ttl'],
@@ -232,8 +249,9 @@ class DnsimpleProvider(BaseProvider):
 
         return self._zone_records[zone.name]
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: name=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
 
         values = defaultdict(lambda: defaultdict(list))
         for record in self.zone_records(zone):
@@ -250,7 +268,8 @@ class DnsimpleProvider(BaseProvider):
         for name, types in values.items():
             for _type, records in types.items():
                 data_for = getattr(self, '_data_for_{}'.format(_type))
-                record = Record.new(zone, name, data_for(_type, records))
+                record = Record.new(zone, name, data_for(_type, records),
+                                    source=self, lenient=lenient)
                 zone.add_record(record)
 
         self.log.info('populate:   found %s records',
@@ -271,6 +290,16 @@ class DnsimpleProvider(BaseProvider):
     _params_for_SPF = _params_for_multiple
     _params_for_TXT = _params_for_multiple
 
+    def _params_for_CAA(self, record):
+        for value in record.values:
+            yield {
+                'content': '{} {} "{}"'.format(value.flags, value.tag,
+                                               value.value),
+                'name': record.name,
+                'ttl': record.ttl,
+                'type': record._type
+            }
+
     def _params_for_single(self, record):
         yield {
             'content': record.value,
@@ -286,9 +315,9 @@ class DnsimpleProvider(BaseProvider):
     def _params_for_MX(self, record):
         for value in record.values:
             yield {
-                'content': value.value,
+                'content': value.exchange,
                 'name': record.name,
-                'priority': value.priority,
+                'priority': value.preference,
                 'ttl': record.ttl,
                 'type': record._type
             }

octodns/provider/dyn.py

@@ -106,10 +106,12 @@ class DynProvider(BaseProvider):
     than one account active at a time. See DynProvider._check_dyn_sess for some
     related bits.
     '''
+
     RECORDS_TO_TYPE = {
         'a_records': 'A',
         'aaaa_records': 'AAAA',
         'alias_records': 'ALIAS',
+        'caa_records': 'CAA',
         'cname_records': 'CNAME',
         'mx_records': 'MX',
         'naptr_records': 'NAPTR',
@@ -121,6 +123,7 @@ class DynProvider(BaseProvider):
         'txt_records': 'TXT',
     }
     TYPE_TO_RECORDS = {v: k for k, v in RECORDS_TO_TYPE.items()}
+    SUPPORTS = set(TYPE_TO_RECORDS.keys())
 
     # https://help.dyn.com/predefined-geotm-regions-groups/
     REGION_CODES = {
@@ -192,6 +195,14 @@ class DynProvider(BaseProvider):
             'value': record.alias
         }
 
+    def _data_for_CAA(self, _type, records):
+        return {
+            'type': _type,
+            'ttl': records[0].ttl,
+            'values': [{'flags': r.flags, 'tag': r.tag, 'value': r.value}
+                       for r in records],
+        }
+
     def _data_for_CNAME(self, _type, records):
         record = records[0]
         return {
@@ -204,7 +215,7 @@ class DynProvider(BaseProvider):
         return {
             'type': _type,
             'ttl': records[0].ttl,
-            'values': [{'priority': r.preference, 'value': r.exchange}
+            'values': [{'preference': r.preference, 'exchange': r.exchange}
                        for r in records],
         }
 
@@ -336,8 +347,10 @@ class DynProvider(BaseProvider):
 
         return td_records
 
-    def populate(self, zone, target=False):
-        self.log.info('populate: zone=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+
         before = len(zone.records)
 
         self._check_dyn_sess()
@@ -362,7 +375,8 @@ class DynProvider(BaseProvider):
                 for _type, records in types.items():
                     data_for = getattr(self, '_data_for_{}'.format(_type))
                     data = data_for(_type, records)
-                    record = Record.new(zone, name, data, source=self)
+                    record = Record.new(zone, name, data, source=self,
+                                        lenient=lenient)
                     if record not in td_records:
                         zone.add_record(record)
 
@@ -377,6 +391,13 @@ class DynProvider(BaseProvider):
 
     _kwargs_for_AAAA = _kwargs_for_A
 
+    def _kwargs_for_CAA(self, record):
+        return [{
+            'flags': v.flags,
+            'tag': v.tag,
+            'value': v.value,
+        } for v in record.values]
+
     def _kwargs_for_CNAME(self, record):
         return [{
             'cname': record.value,
@@ -395,8 +416,8 @@ class DynProvider(BaseProvider):
 
     def _kwargs_for_MX(self, record):
         return [{
-            'preference': v.priority,
-            'exchange': v.value,
+            'preference': v.preference,
+            'exchange': v.exchange,
             'ttl': record.ttl,
         } for v in record.values]
 

octodns/provider/ns1.py

@@ -7,7 +7,8 @@ from __future__ import absolute_import, division, print_function, \
 
 from logging import getLogger
 from nsone import NSONE
-from nsone.rest.errors import ResourceException
+from nsone.rest.errors import RateLimitException, ResourceException
+from time import sleep
 
 from ..record import Record
 from .base import BaseProvider
@@ -22,6 +23,9 @@ class Ns1Provider(BaseProvider):
         api_key: env/NS1_API_KEY
     '''
     SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS',
+                    'PTR', 'SPF', 'SRV', 'TXT'))
+
     ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found'
 
     def __init__(self, id, api_key, *args, **kwargs):
@@ -30,9 +34,6 @@ class Ns1Provider(BaseProvider):
         super(Ns1Provider, self).__init__(id, *args, **kwargs)
         self._client = NSONE(apiKey=api_key)
 
-    def supports(self, record):
-        return record._type != 'SSHFP'
-
     def _data_for_A(self, _type, record):
         return {
             'ttl': record['ttl'],
@@ -41,8 +42,31 @@ class Ns1Provider(BaseProvider):
         }
 
     _data_for_AAAA = _data_for_A
-    _data_for_SPF = _data_for_A
-    _data_for_TXT = _data_for_A
+
+    def _data_for_SPF(self, _type, record):
+        values = [v.replace(';', '\;') for v in record['short_answers']]
+        return {
+            'ttl': record['ttl'],
+            'type': _type,
+            'values': values
+        }
+
+    _data_for_TXT = _data_for_SPF
+
+    def _data_for_CAA(self, _type, record):
+        values = []
+        for answer in record['short_answers']:
+            flags, tag, value = answer.split(' ', 2)
+            values.append({
+                'flags': flags,
+                'tag': tag,
+                'value': value,
+            })
+        return {
+            'ttl': record['ttl'],
+            'type': _type,
+            'values': values,
+        }
 
     def _data_for_CNAME(self, _type, record):
         return {
@@ -57,10 +81,10 @@ class Ns1Provider(BaseProvider):
     def _data_for_MX(self, _type, record):
         values = []
         for answer in record['short_answers']:
-            priority, value = answer.split(' ', 1)
+            preference, exchange = answer.split(' ', 1)
             values.append({
-                'priority': priority,
-                'value': value,
+                'preference': preference,
+                'exchange': exchange,
             })
         return {
             'ttl': record['ttl'],
@@ -111,8 +135,9 @@ class Ns1Provider(BaseProvider):
             'values': values,
         }
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: name=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
 
         try:
             nsone_zone = self._client.loadZone(zone.name[:-1])
@@ -127,7 +152,8 @@ class Ns1Provider(BaseProvider):
             _type = record['type']
             data_for = getattr(self, '_data_for_{}'.format(_type))
             name = zone.hostname_from_fqdn(record['domain'])
-            record = Record.new(zone, name, data_for(_type, record))
+            record = Record.new(zone, name, data_for(_type, record),
+                                source=self, lenient=lenient)
             zone.add_record(record)
 
         self.log.info('populate:   found %s records',
@@ -138,8 +164,19 @@ class Ns1Provider(BaseProvider):
 
     _params_for_AAAA = _params_for_A
     _params_for_NS = _params_for_A
-    _params_for_SPF = _params_for_A
-    _params_for_TXT = _params_for_A
+
+    def _params_for_SPF(self, record):
+        # NS1 seems to be the only provider that doesn't want things escaped in
+        # values so we have to strip them here and add them when going the
+        # other way
+        values = [v.replace('\;', ';') for v in record.values]
+        return {'answers': values, 'ttl': record.ttl}
+
+    _params_for_TXT = _params_for_SPF
+
+    def _params_for_CAA(self, record):
+        values = [(v.flags, v.tag, v.value) for v in record.values]
+        return {'answers': values, 'ttl': record.ttl}
 
     def _params_for_CNAME(self, record):
         return {'answers': [record.value], 'ttl': record.ttl}
@@ -148,7 +185,7 @@ class Ns1Provider(BaseProvider):
     _params_for_PTR = _params_for_CNAME
 
     def _params_for_MX(self, record):
-        values = [(v.priority, v.value) for v in record.values]
+        values = [(v.preference, v.exchange) for v in record.values]
         return {'answers': values, 'ttl': record.ttl}
 
     def _params_for_NAPTR(self, record):
@@ -169,7 +206,14 @@ class Ns1Provider(BaseProvider):
         name = self._get_name(new)
         _type = new._type
         params = getattr(self, '_params_for_{}'.format(_type))(new)
-        getattr(nsone_zone, 'add_{}'.format(_type))(name, **params)
+        meth = getattr(nsone_zone, 'add_{}'.format(_type))
+        try:
+            meth(name, **params)
+        except RateLimitException as e:
+            self.log.warn('_apply_Create: rate limit encountered, pausing '
+                          'for %ds and trying again', e.period)
+            sleep(e.period)
+            meth(name, **params)
 
     def _apply_Update(self, nsone_zone, change):
         existing = change.existing
@@ -178,14 +222,26 @@ class Ns1Provider(BaseProvider):
         record = nsone_zone.loadRecord(name, _type)
         new = change.new
         params = getattr(self, '_params_for_{}'.format(_type))(new)
-        record.update(**params)
+        try:
+            record.update(**params)
+        except RateLimitException as e:
+            self.log.warn('_apply_Update: rate limit encountered, pausing '
+                          'for %ds and trying again', e.period)
+            sleep(e.period)
+            record.update(**params)
 
     def _apply_Delete(self, nsone_zone, change):
         existing = change.existing
         name = self._get_name(existing)
         _type = existing._type
         record = nsone_zone.loadRecord(name, _type)
-        record.delete()
+        try:
+            record.delete()
+        except RateLimitException as e:
+            self.log.warn('_apply_Delete: rate limit encountered, pausing '
+                          'for %ds and trying again', e.period)
+            sleep(e.period)
+            record.delete()
 
     def _apply(self, plan):
         desired = plan.desired

octodns/provider/powerdns.py

@@ -14,13 +14,17 @@ from .base import BaseProvider
 
 class PowerDnsBaseProvider(BaseProvider):
     SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS',
+                    'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT'))
     TIMEOUT = 5
 
-    def __init__(self, id, host, api_key, port=8081, *args, **kwargs):
+    def __init__(self, id, host, api_key, port=8081, scheme="http", *args,
+                 **kwargs):
         super(PowerDnsBaseProvider, self).__init__(id, *args, **kwargs)
 
         self.host = host
         self.port = port
+        self.scheme = scheme
 
         sess = Session()
         sess.headers.update({'X-API-Key': api_key})
@@ -29,8 +33,8 @@ class PowerDnsBaseProvider(BaseProvider):
     def _request(self, method, path, data=None):
         self.log.debug('_request: method=%s, path=%s', method, path)
 
-        url = 'http://{}:{}/api/v1/servers/localhost/{}' \
-            .format(self.host, self.port, path)
+        url = '{}://{}:{}/api/v1/servers/localhost/{}' \
+            .format(self.scheme, self.host, self.port, path)
         resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT)
         self.log.debug('_request:   status=%d', resp.status_code)
         resp.raise_for_status()
@@ -57,6 +61,21 @@ class PowerDnsBaseProvider(BaseProvider):
     _data_for_AAAA = _data_for_multiple
     _data_for_NS = _data_for_multiple
 
+    def _data_for_CAA(self, rrset):
+        values = []
+        for record in rrset['records']:
+            flags, tag, value = record['content'].split(' ', 2)
+            values.append({
+                'flags': flags,
+                'tag': tag,
+                'value': value[1:-1],
+            })
+        return {
+            'type': rrset['type'],
+            'values': values,
+            'ttl': rrset['ttl']
+        }
+
     def _data_for_single(self, rrset):
         return {
             'type': rrset['type'],
@@ -81,10 +100,10 @@ class PowerDnsBaseProvider(BaseProvider):
     def _data_for_MX(self, rrset):
         values = []
         for record in rrset['records']:
-            priority, value = record['content'].split(' ', 1)
+            preference, exchange = record['content'].split(' ', 1)
             values.append({
-                'priority': priority,
-                'value': value,
+                'preference': preference,
+                'exchange': exchange,
             })
         return {
             'type': rrset['type'],
@@ -144,8 +163,9 @@ class PowerDnsBaseProvider(BaseProvider):
             'ttl': rrset['ttl']
         }
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: name=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
 
         resp = None
         try:
@@ -175,7 +195,7 @@ class PowerDnsBaseProvider(BaseProvider):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 record_name = zone.hostname_from_fqdn(rrset['name'])
                 record = Record.new(zone, record_name, data_for(rrset),
-                                    source=self)
+                                    source=self, lenient=lenient)
                 zone.add_record(record)
 
         self.log.info('populate:   found %s records',
@@ -189,6 +209,12 @@ class PowerDnsBaseProvider(BaseProvider):
     _records_for_AAAA = _records_for_multiple
     _records_for_NS = _records_for_multiple
 
+    def _records_for_CAA(self, record):
+        return [{
+            'content': '{} {} "{}"'.format(v.flags, v.tag, v.value),
+            'disabled': False
+        } for v in record.values]
+
     def _records_for_single(self, record):
         return [{'content': record.value, 'disabled': False}]
 
@@ -205,7 +231,7 @@ class PowerDnsBaseProvider(BaseProvider):
 
     def _records_for_MX(self, record):
         return [{
-            'content': '{} {}'.format(v.priority, v.value),
+            'content': '{} {}'.format(v.preference, v.exchange),
             'disabled': False
         } for v in record.values]
 

octodns/provider/route53.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
 from boto3 import client
+from botocore.config import Config
 from collections import defaultdict
 from incf.countryutils.transformations import cca_to_ctca2
 from uuid import uuid4
@@ -16,27 +17,71 @@ from ..record import Record, Update
 from .base import BaseProvider
 
 
+octal_re = re.compile(r'\\(\d\d\d)')
+
+
+def _octal_replace(s):
+    # See http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/
+    #     DomainNameFormat.html
+    return octal_re.sub(lambda m: chr(int(m.group(1), 8)), s)
+
+
 class _Route53Record(object):
 
-    def __init__(self, fqdn, _type, ttl, record=None, values=None, geo=None,
-                 health_check_id=None):
-        self.fqdn = fqdn
-        self._type = _type
-        self.ttl = ttl
-        # From here on things are a little ugly, it works, but would be nice to
-        # clean up someday.
-        if record:
-            values_for = getattr(self, '_values_for_{}'.format(self._type))
-            self.values = values_for(record)
+    @classmethod
+    def new(self, provider, record, creating):
+        ret = set()
+        if getattr(record, 'geo', False):
+            ret.add(_Route53GeoDefault(provider, record, creating))
+            for ident, geo in record.geo.items():
+                ret.add(_Route53GeoRecord(provider, record, ident, geo,
+                                          creating))
         else:
-            self.values = values
-        self.geo = geo
-        self.health_check_id = health_check_id
-        self.is_geo_default = False
+            ret.add(_Route53Record(provider, record, creating))
+        return ret
 
-    @property
-    def _geo_code(self):
-        return getattr(self.geo, 'code', '')
+    def __init__(self, provider, record, creating):
+        self.fqdn = record.fqdn
+        self._type = record._type
+        self.ttl = record.ttl
+
+        values_for = getattr(self, '_values_for_{}'.format(self._type))
+        self.values = values_for(record)
+
+    def mod(self, action):
+        return {
+            'Action': action,
+            'ResourceRecordSet': {
+                'Name': self.fqdn,
+                'ResourceRecords': [{'Value': v} for v in self.values],
+                'TTL': self.ttl,
+                'Type': self._type,
+            }
+        }
+
+    # NOTE: we're using __hash__ and __cmp__ methods that consider
+    # _Route53Records equivalent if they have the same class, fqdn, and _type.
+    # Values are ignored. This is usful when computing diffs/changes.
+
+    def __hash__(self):
+        'sub-classes should never use this method'
+        return '{}:{}'.format(self.fqdn, self._type).__hash__()
+
+    def __cmp__(self, other):
+        '''sub-classes should call up to this and return its value if non-zero.
+        When it's zero they should compute their own __cmp__'''
+        if self.__class__ != other.__class__:
+            return cmp(self.__class__, other.__class__)
+        elif self.fqdn != other.fqdn:
+            return cmp(self.fqdn, other.fqdn)
+        elif self._type != other._type:
+            return cmp(self._type, other._type)
+        # We're ignoring ttl, it's not an actual differentiator
+        return 0
+
+    def __repr__(self):
+        return '_Route53Record<{} {} {} {}>'.format(self.fqdn, self._type,
+                                                    self.ttl, self.values)
 
     def _values_for_values(self, record):
         return record.values
@@ -45,6 +90,10 @@ class _Route53Record(object):
     _values_for_AAAA = _values_for_values
     _values_for_NS = _values_for_values
 
+    def _values_for_CAA(self, record):
+        return ['{} {} "{}"'.format(v.flags, v.tag, v.value)
+                for v in record.values]
+
     def _values_for_value(self, record):
         return [record.value]
 
@@ -52,7 +101,8 @@ class _Route53Record(object):
     _values_for_PTR = _values_for_value
 
     def _values_for_MX(self, record):
-        return ['{} {}'.format(v.priority, v.value) for v in record.values]
+        return ['{} {}'.format(v.preference, v.exchange)
+                for v in record.values]
 
     def _values_for_NAPTR(self, record):
         return ['{} {} "{}" "{}" "{}" {}'
@@ -75,68 +125,91 @@ class _Route53Record(object):
                                      v.target)
                 for v in record.values]
 
+
+class _Route53GeoDefault(_Route53Record):
+
     def mod(self, action):
+        return {
+            'Action': action,
+            'ResourceRecordSet': {
+                'Name': self.fqdn,
+                'GeoLocation': {
+                    'CountryCode': '*'
+                },
+                'ResourceRecords': [{'Value': v} for v in self.values],
+                'SetIdentifier': 'default',
+                'TTL': self.ttl,
+                'Type': self._type,
+            }
+        }
+
+    def __hash__(self):
+        return '{}:{}:default'.format(self.fqdn, self._type).__hash__()
+
+    def __repr__(self):
+        return '_Route53GeoDefault<{} {} {} {}>'.format(self.fqdn, self._type,
+                                                        self.ttl, self.values)
+
+
+class _Route53GeoRecord(_Route53Record):
+
+    def __init__(self, provider, record, ident, geo, creating):
+        super(_Route53GeoRecord, self).__init__(provider, record, creating)
+        self.geo = geo
+
+        self.health_check_id = provider.get_health_check_id(record, ident,
+                                                            geo, creating)
+
+    def mod(self, action):
+        geo = self.geo
         rrset = {
             'Name': self.fqdn,
-            'Type': self._type,
-            'TTL': self.ttl,
-            'ResourceRecords': [{'Value': v} for v in self.values],
-        }
-        if self.is_geo_default:
-            rrset['GeoLocation'] = {
+            'GeoLocation': {
                 'CountryCode': '*'
+            },
+            'ResourceRecords': [{'Value': v} for v in geo.values],
+            'SetIdentifier': geo.code,
+            'TTL': self.ttl,
+            'Type': self._type,
+        }
+
+        if self.health_check_id:
+            rrset['HealthCheckId'] = self.health_check_id
+
+        if geo.subdivision_code:
+            rrset['GeoLocation'] = {
+                'CountryCode': geo.country_code,
+                'SubdivisionCode': geo.subdivision_code
+            }
+        elif geo.country_code:
+            rrset['GeoLocation'] = {
+                'CountryCode': geo.country_code
+            }
+        else:
+            rrset['GeoLocation'] = {
+                'ContinentCode': geo.continent_code
             }
-            rrset['SetIdentifier'] = 'default'
-        elif self.geo:
-            geo = self.geo
-            rrset['SetIdentifier'] = geo.code
-            if self.health_check_id:
-                rrset['HealthCheckId'] = self.health_check_id
-            if geo.subdivision_code:
-                rrset['GeoLocation'] = {
-                    'CountryCode': geo.country_code,
-                    'SubdivisionCode': geo.subdivision_code
-                }
-            elif geo.country_code:
-                rrset['GeoLocation'] = {
-                    'CountryCode': geo.country_code
-                }
-            else:
-                rrset['GeoLocation'] = {
-                    'ContinentCode': geo.continent_code
-                }
 
         return {
             'Action': action,
             'ResourceRecordSet': rrset,
         }
 
-    # NOTE: we're using __hash__ and __cmp__ methods that consider
-    # _Route53Records equivalent if they have the same fqdn, _type, and
-    # geo.ident. Values are ignored. This is usful when computing
-    # diffs/changes.
-
     def __hash__(self):
         return '{}:{}:{}'.format(self.fqdn, self._type,
-                                 self._geo_code).__hash__()
+                                 self.geo.code).__hash__()
 
     def __cmp__(self, other):
-        return 0 if (self.fqdn == other.fqdn and
-                     self._type == other._type and
-                     self._geo_code == other._geo_code) else 1
+        ret = super(_Route53GeoRecord, self).__cmp__(other)
+        if ret != 0:
+            return ret
+        return cmp(self.geo.code, other.geo.code)
 
     def __repr__(self):
-        return '_Route53Record<{} {:>5} {:8} {}>' \
-            .format(self.fqdn, self._type, self._geo_code, self.values)
-
-
-octal_re = re.compile(r'\\(\d\d\d)')
-
-
-def _octal_replace(s):
-    # See http://docs.aws.amazon.com/Route53/latest/DeveloperGuide/
-    #     DomainNameFormat.html
-    return octal_re.sub(lambda m: chr(int(m.group(1), 8)), s)
+        return '_Route53GeoRecord<{} {} {} {} {}>'.format(self.fqdn,
+                                                          self._type, self.ttl,
+                                                          self.geo.code,
+                                                          self.values)
 
 
 class Route53Provider(BaseProvider):
@@ -153,28 +226,35 @@ class Route53Provider(BaseProvider):
     In general the account used will need full permissions on Route53.
     '''
     SUPPORTS_GEO = True
+    SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR',
+                    'SPF', 'SRV', 'TXT'))
 
     # This should be bumped when there are underlying changes made to the
     # health check config.
     HEALTH_CHECK_VERSION = '0000'
 
     def __init__(self, id, access_key_id, secret_access_key, max_changes=1000,
-                 *args, **kwargs):
+                 client_max_attempts=None, *args, **kwargs):
         self.max_changes = max_changes
         self.log = logging.getLogger('Route53Provider[{}]'.format(id))
         self.log.debug('__init__: id=%s, access_key_id=%s, '
                        'secret_access_key=***', id, access_key_id)
         super(Route53Provider, self).__init__(id, *args, **kwargs)
+
+        config = None
+        if client_max_attempts is not None:
+            self.log.info('__init__: setting max_attempts to %d',
+                          client_max_attempts)
+            config = Config(retries={'max_attempts': client_max_attempts})
+
         self._conn = client('route53', aws_access_key_id=access_key_id,
-                            aws_secret_access_key=secret_access_key)
+                            aws_secret_access_key=secret_access_key,
+                            config=config)
 
         self._r53_zones = None
         self._r53_rrsets = {}
         self._health_checks = None
 
-    def supports(self, record):
-        return record._type not in ('ALIAS', 'SSHFP')
-
     @property
     def r53_zones(self):
         if self._r53_zones is None:
@@ -183,7 +263,7 @@ class Route53Provider(BaseProvider):
             more = True
             start = {}
             while more:
-                resp = self._conn.list_hosted_zones()
+                resp = self._conn.list_hosted_zones(**start)
                 for z in resp['HostedZones']:
                     zones[z['Name']] = z['Id']
                 more = resp['IsTruncated']
@@ -243,6 +323,21 @@ class Route53Provider(BaseProvider):
     _data_for_A = _data_for_geo
     _data_for_AAAA = _data_for_geo
 
+    def _data_for_CAA(self, rrset):
+        values = []
+        for rr in rrset['ResourceRecords']:
+            flags, tag, value = rr['Value'].split(' ')
+            values.append({
+                'flags': flags,
+                'tag': tag,
+                'value': value[1:-1],
+            })
+        return {
+            'type': rrset['Type'],
+            'values': values,
+            'ttl': int(rrset['TTL'])
+        }
+
     def _data_for_single(self, rrset):
         return {
             'type': rrset['Type'],
@@ -269,10 +364,10 @@ class Route53Provider(BaseProvider):
     def _data_for_MX(self, rrset):
         values = []
         for rr in rrset['ResourceRecords']:
-            priority, value = rr['Value'].split(' ')
+            preference, exchange = rr['Value'].split(' ')
             values.append({
-                'priority': priority,
-                'value': value,
+                'preference': preference,
+                'exchange': exchange,
             })
         return {
             'type': rrset['Type'],
@@ -352,8 +447,10 @@ class Route53Provider(BaseProvider):
 
         return self._r53_rrsets[zone_id]
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: name=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+
         before = len(zone.records)
 
         zone_id = self._get_zone_id(zone.name)
@@ -383,7 +480,8 @@ class Route53Provider(BaseProvider):
                         data['geo'] = geo
                     else:
                         data = data[0]
-                    record = Record.new(zone, name, data, source=self)
+                    record = Record.new(zone, name, data, source=self,
+                                        lenient=lenient)
                     zone.add_record(record)
 
         self.log.info('populate:   found %s records',
@@ -391,7 +489,7 @@ class Route53Provider(BaseProvider):
 
     def _gen_mods(self, action, records):
         '''
-        Turns `_Route53Record`s in to `change_resource_record_sets` `Changes`
+        Turns `_Route53*`s in to `change_resource_record_sets` `Changes`
         '''
         return [r.mod(action) for r in records]
 
@@ -420,14 +518,14 @@ class Route53Provider(BaseProvider):
         # We've got a cached version use it
         return self._health_checks
 
-    def _get_health_check_id(self, record, ident, geo, create):
+    def get_health_check_id(self, record, ident, geo, create):
         # fqdn & the first value are special, we use them to match up health
         # checks to their records. Route53 health checks check a single ip and
         # we're going to assume that ips are interchangeable to avoid
         # health-checking each one independently
         fqdn = record.fqdn
         first_value = geo.values[0]
-        self.log.debug('_get_health_check_id: fqdn=%s, type=%s, geo=%s, '
+        self.log.debug('get_health_check_id: fqdn=%s, type=%s, geo=%s, '
                        'first_value=%s', fqdn, record._type, ident,
                        first_value)
 
@@ -473,7 +571,7 @@ class Route53Provider(BaseProvider):
         # store the new health check so that we'll be able to find it in the
         # future
         self._health_checks[id] = health_check
-        self.log.info('_get_health_check_id: created id=%s, host=%s, '
+        self.log.info('get_health_check_id: created id=%s, host=%s, '
                       'first_value=%s', id, host, first_value)
         return id
 
@@ -482,8 +580,9 @@ class Route53Provider(BaseProvider):
         # Find the health checks we're using for the new route53 records
         in_use = set()
         for r in new:
-            if r.health_check_id:
-                in_use.add(r.health_check_id)
+            hc_id = getattr(r, 'health_check_id', False)
+            if hc_id:
+                in_use.add(hc_id)
         self.log.debug('_gc_health_checks:   in_use=%s', in_use)
         # Now we need to run through ALL the health checks looking for those
         # that apply to this record, deleting any that do and are no longer in
@@ -502,23 +601,9 @@ class Route53Provider(BaseProvider):
 
     def _gen_records(self, record, creating=False):
         '''
-        Turns an octodns.Record into one or more `_Route53Record`s
+        Turns an octodns.Record into one or more `_Route53*`s
         '''
-        records = set()
-        base = _Route53Record(record.fqdn, record._type, record.ttl,
-                              record=record)
-        records.add(base)
-        if getattr(record, 'geo', False):
-            base.is_geo_default = True
-            for ident, geo in record.geo.items():
-                health_check_id = self._get_health_check_id(record, ident, geo,
-                                                            creating)
-                records.add(_Route53Record(record.fqdn, record._type,
-                                           record.ttl, values=geo.values,
-                                           geo=geo,
-                                           health_check_id=health_check_id))
-
-        return records
+        return _Route53Record.new(self, record, creating)
 
     def _mod_Create(self, change):
         # New is the stuff that needs to be created
@@ -544,24 +629,11 @@ class Route53Provider(BaseProvider):
         # things that haven't actually changed, but that's for another day.
         # We can't use set math here b/c we won't be able to control which of
         # the two objects will be in the result and we need to ensure it's the
-        # new one and we have to include some special handling when converting
-        # to/from a GEO enabled record
+        # new one.
         upserts = set()
-        existing_records = {r: r for r in existing_records}
         for new_record in new_records:
-            try:
-                existing_record = existing_records[new_record]
-                if new_record.is_geo_default != existing_record.is_geo_default:
-                    # going from normal to geo or geo to normal, need a delete
-                    # and create
-                    deletes.add(existing_record)
-                    creates.add(new_record)
-                else:
-                    # just an update
-                    upserts.add(new_record)
-            except KeyError:
-                # Completely new record, ignore
-                pass
+            if new_record in existing_records:
+                upserts.add(new_record)
 
         return self._gen_mods('DELETE', deletes) + \
             self._gen_mods('CREATE', creates) + \

octodns/provider/yaml.py

@@ -26,19 +26,29 @@ class YamlProvider(BaseProvider):
         # The ttl to use for records when not specified in the data
         # (optional, default 3600)
         default_ttl: 3600
+        # Whether or not to enforce sorting order on the yaml config
+        # (optional, default True)
+        enforce_order: True
     '''
     SUPPORTS_GEO = True
+    SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR',
+                   'SSHFP', 'SPF', 'SRV', 'TXT'))
 
-    def __init__(self, id, directory, default_ttl=3600, *args, **kwargs):
+    def __init__(self, id, directory, default_ttl=3600, enforce_order=True,
+                 *args, **kwargs):
         self.log = logging.getLogger('YamlProvider[{}]'.format(id))
-        self.log.debug('__init__: id=%s, directory=%s, default_ttl=%d', id,
-                       directory, default_ttl)
+        self.log.debug('__init__: id=%s, directory=%s, default_ttl=%d, '
+                       'enforce_order=%d', id, directory, default_ttl,
+                       enforce_order)
         super(YamlProvider, self).__init__(id, *args, **kwargs)
         self.directory = directory
         self.default_ttl = default_ttl
+        self.enforce_order = enforce_order
+
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: zone=%s, target=%s', zone.name, target)
         if target:
             # When acting as a target we ignore any existing records so that we
             # create a completely new copy
@@ -47,7 +57,7 @@ class YamlProvider(BaseProvider):
         before = len(zone.records)
         filename = join(self.directory, '{}yaml'.format(zone.name))
         with open(filename, 'r') as fh:
-            yaml_data = safe_load(fh)
+            yaml_data = safe_load(fh, enforce_order=self.enforce_order)
             if yaml_data:
                 for name, data in yaml_data.items():
                     if not isinstance(data, list):
@@ -55,7 +65,8 @@ class YamlProvider(BaseProvider):
                     for d in data:
                         if 'ttl' not in d:
                             d['ttl'] = self.default_ttl
-                        record = Record.new(zone, name, d, source=self)
+                        record = Record.new(zone, name, d, source=self,
+                                            lenient=lenient)
                         zone.add_record(record)
 
         self.log.info('populate:   found %s records',

octodns/record.py

@@ -54,51 +54,64 @@ class Delete(Change):
         return 'Delete {}'.format(self.existing)
 
 
-_unescaped_semicolon_re = re.compile(r'\w;')
+class ValidationError(Exception):
+
+    @classmethod
+    def build_message(cls, fqdn, reasons):
+        return 'Invalid record {}\n  - {}'.format(fqdn, '\n  - '.join(reasons))
+
+    def __init__(self, fqdn, reasons):
+        super(Exception, self).__init__(self.build_message(fqdn, reasons))
+        self.fqdn = fqdn
+        self.reasons = reasons
 
 
 class Record(object):
     log = getLogger('Record')
 
     @classmethod
-    def new(cls, zone, name, data, source=None):
+    def new(cls, zone, name, data, source=None, lenient=False):
+        fqdn = '{}.{}'.format(name, zone.name) if name else zone.name
         try:
             _type = data['type']
         except KeyError:
-            fqdn = '{}.{}'.format(name, zone.name) if name else zone.name
             raise Exception('Invalid record {}, missing type'.format(fqdn))
         try:
-            _type = {
+            _class = {
                 'A': ARecord,
                 'AAAA': AaaaRecord,
                 'ALIAS': AliasRecord,
-                # cert
+                'CAA': CaaRecord,
                 'CNAME': CnameRecord,
-                # dhcid
-                # dname
-                # dnskey
-                # ds
-                # ipseckey
-                # key
-                # kx
-                # loc
                 'MX': MxRecord,
                 'NAPTR': NaptrRecord,
                 'NS': NsRecord,
-                # nsap
                 'PTR': PtrRecord,
-                # px
-                # rp
-                # soa - would it even make sense?
                 'SPF': SpfRecord,
                 'SRV': SrvRecord,
                 'SSHFP': SshfpRecord,
                 'TXT': TxtRecord,
-                # url
             }[_type]
         except KeyError:
             raise Exception('Unknown record type: "{}"'.format(_type))
-        return _type(zone, name, data, source=source)
+        reasons = _class.validate(name, data)
+        if reasons:
+            if lenient:
+                cls.log.warn(ValidationError.build_message(fqdn, reasons))
+            else:
+                raise ValidationError(fqdn, reasons)
+        return _class(zone, name, data, source=source)
+
+    @classmethod
+    def validate(cls, name, data):
+        reasons = []
+        try:
+            ttl = int(data['ttl'])
+            if ttl < 0:
+                reasons.append('invalid ttl')
+        except KeyError:
+            reasons.append('missing ttl')
+        return reasons
 
     def __init__(self, zone, name, data, source=None):
         self.log.debug('__init__: zone.name=%s, type=%11s, name=%s', zone.name,
@@ -106,11 +119,8 @@ class Record(object):
         self.zone = zone
         # force everything lower-case just to be safe
         self.name = str(name).lower() if name else name
-        try:
-            self.ttl = int(data['ttl'])
-        except KeyError:
-            raise Exception('Invalid record {}, missing ttl'.format(self.fqdn))
         self.source = source
+        self.ttl = int(data['ttl'])
 
         octodns = data.get('octodns', {})
         self.ignored = octodns.get('ignored', False)
@@ -154,11 +164,17 @@ class GeoValue(object):
     geo_re = re.compile(r'^(?P<continent_code>\w\w)(-(?P<country_code>\w\w)'
                         r'(-(?P<subdivision_code>\w\w))?)?$')
 
-    def __init__(self, geo, values):
-        match = self.geo_re.match(geo)
+    @classmethod
+    def _validate_geo(cls, code):
+        reasons = []
+        match = cls.geo_re.match(code)
         if not match:
-            raise Exception('Invalid geo "{}"'.format(geo))
+            reasons.append('invalid geo "{}"'.format(code))
+        return reasons
+
+    def __init__(self, geo, values):
         self.code = geo
+        match = self.geo_re.match(geo)
         self.continent_code = match.group('continent_code')
         self.country_code = match.group('country_code')
         self.subdivision_code = match.group('subdivision_code')
@@ -185,16 +201,29 @@ class GeoValue(object):
 
 class _ValuesMixin(object):
 
-    def __init__(self, zone, name, data, source=None):
-        super(_ValuesMixin, self).__init__(zone, name, data, source=source)
+    @classmethod
+    def validate(cls, name, data):
+        reasons = super(_ValuesMixin, cls).validate(name, data)
+        values = []
         try:
             values = data['values']
         except KeyError:
             try:
                 values = [data['value']]
             except KeyError:
-                raise Exception('Invalid record {}, missing value(s)'
-                                .format(self.fqdn))
+                reasons.append('missing value(s)')
+
+        for value in values:
+            reasons.extend(cls._validate_value(value))
+
+        return reasons
+
+    def __init__(self, zone, name, data, source=None):
+        super(_ValuesMixin, self).__init__(zone, name, data, source=source)
+        try:
+            values = data['values']
+        except KeyError:
+            values = [data['value']]
         self.values = sorted(self._process_values(values))
 
     def changes(self, other, target):
@@ -212,9 +241,10 @@ class _ValuesMixin(object):
         return ret
 
     def __repr__(self):
+        values = "['{}']".format("', '".join([str(v) for v in self.values]))
         return '<{} {} {}, {}, {}>'.format(self.__class__.__name__,
                                            self._type, self.ttl,
-                                           self.fqdn, self.values)
+                                           self.fqdn, values)
 
 
 class _GeoMixin(_ValuesMixin):
@@ -224,6 +254,21 @@ class _GeoMixin(_ValuesMixin):
     Must be included before `Record`.
     '''
 
+    @classmethod
+    def validate(cls, name, data):
+        reasons = super(_GeoMixin, cls).validate(name, data)
+        try:
+            geo = dict(data['geo'])
+            # TODO: validate legal codes
+            for code, values in geo.items():
+                reasons.extend(GeoValue._validate_geo(code))
+                for value in values:
+                    reasons.extend(cls._validate_value(value))
+        except KeyError:
+            pass
+        return reasons
+
+    # TODO: support 'value' as well
     # TODO: move away from "data" hash to strict params, it's kind of leaking
     # the yaml implementation into here and then forcing it back out into
     # non-yaml providers during input
@@ -233,9 +278,8 @@ class _GeoMixin(_ValuesMixin):
             self.geo = dict(data['geo'])
         except KeyError:
             self.geo = {}
-        for k, vs in self.geo.items():
-            vs = sorted(self._process_values(vs))
-            self.geo[k] = GeoValue(k, vs)
+        for code, values in self.geo.items():
+            self.geo[code] = GeoValue(code, values)
 
     def _data(self):
         ret = super(_GeoMixin, self)._data()
@@ -264,41 +308,52 @@ class _GeoMixin(_ValuesMixin):
 class ARecord(_GeoMixin, Record):
     _type = 'A'
 
+    @classmethod
+    def _validate_value(self, value):
+        reasons = []
+        try:
+            IPv4Address(unicode(value))
+        except Exception:
+            reasons.append('invalid ip address "{}"'.format(value))
+        return reasons
+
     def _process_values(self, values):
-        for ip in values:
-            try:
-                IPv4Address(unicode(ip))
-            except Exception:
-                raise Exception('Invalid record {}, value {} not a valid ip'
-                                .format(self.fqdn, ip))
         return values
 
 
 class AaaaRecord(_GeoMixin, Record):
     _type = 'AAAA'
 
+    @classmethod
+    def _validate_value(self, value):
+        reasons = []
+        try:
+            IPv6Address(unicode(value))
+        except Exception:
+            reasons.append('invalid ip address "{}"'.format(value))
+        return reasons
+
     def _process_values(self, values):
-        ret = []
-        for ip in values:
-            try:
-                IPv6Address(unicode(ip))
-                ret.append(ip.lower())
-            except Exception:
-                raise Exception('Invalid record {}, value {} not a valid ip'
-                                .format(self.fqdn, ip))
-        return ret
+        return values
 
 
 class _ValueMixin(object):
 
-    def __init__(self, zone, name, data, source=None):
-        super(_ValueMixin, self).__init__(zone, name, data, source=source)
+    @classmethod
+    def validate(cls, name, data):
+        reasons = super(_ValueMixin, cls).validate(name, data)
+        value = None
         try:
             value = data['value']
         except KeyError:
-            raise Exception('Invalid record {}, missing value'
-                            .format(self.fqdn))
-        self.value = self._process_value(value)
+            reasons.append('missing value')
+        if value:
+            reasons.extend(cls._validate_value(value))
+        return reasons
+
+    def __init__(self, zone, name, data, source=None):
+        super(_ValueMixin, self).__init__(zone, name, data, source=source)
+        self.value = self._process_value(data['value'])
 
     def changes(self, other, target):
         if self.value != other.value:
@@ -319,62 +374,187 @@ class _ValueMixin(object):
 class AliasRecord(_ValueMixin, Record):
     _type = 'ALIAS'
 
-    def _process_value(self, value):
+    @classmethod
+    def _validate_value(self, value):
+        reasons = []
         if not value.endswith('.'):
-            raise Exception('Invalid record {}, value ({}) missing trailing .'
-                            .format(self.fqdn, value))
+            reasons.append('missing trailing .')
+        return reasons
+
+    def _process_value(self, value):
         return value
 
 
+class CaaValue(object):
+    # https://tools.ietf.org/html/rfc6844#page-5
+
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
+        try:
+            flags = int(value.get('flags', 0))
+            if flags < 0 or flags > 255:
+                reasons.append('invalid flags "{}"'.format(flags))
+        except ValueError:
+            reasons.append('invalid flags "{}"'.format(value['flags']))
+
+        if 'tag' not in value:
+            reasons.append('missing tag')
+        if 'value' not in value:
+            reasons.append('missing value')
+
+        return reasons
+
+    def __init__(self, value):
+        self.flags = int(value.get('flags', 0))
+        self.tag = value['tag']
+        self.value = value['value']
+
+    @property
+    def data(self):
+        return {
+            'flags': self.flags,
+            'tag': self.tag,
+            'value': self.value,
+        }
+
+    def __cmp__(self, other):
+        if self.flags == other.flags:
+            if self.tag == other.tag:
+                return cmp(self.value, other.value)
+            return cmp(self.tag, other.tag)
+        return cmp(self.flags, other.flags)
+
+    def __repr__(self):
+        return '{} {} "{}"'.format(self.flags, self.tag, self.value)
+
+
+class CaaRecord(_ValuesMixin, Record):
+    _type = 'CAA'
+
+    @classmethod
+    def _validate_value(cls, value):
+        return CaaValue._validate_value(value)
+
+    def _process_values(self, values):
+        return [CaaValue(v) for v in values]
+
+
 class CnameRecord(_ValueMixin, Record):
     _type = 'CNAME'
 
-    def _process_value(self, value):
+    @classmethod
+    def validate(cls, name, data):
+        reasons = []
+        if name == '':
+            reasons.append('root CNAME not allowed')
+        reasons.extend(super(CnameRecord, cls).validate(name, data))
+        return reasons
+
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
         if not value.endswith('.'):
-            raise Exception('Invalid record {}, value ({}) missing trailing .'
-                            .format(self.fqdn, value))
-        return value.lower()
+            reasons.append('missing trailing .')
+        return reasons
+
+    def _process_value(self, value):
+        return value
 
 
 class MxValue(object):
 
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
+        try:
+            int(value.get('preference', None) or value['priority'])
+        except KeyError:
+            reasons.append('missing preference')
+        except ValueError:
+            reasons.append('invalid preference "{}"'
+                           .format(value['preference']))
+        exchange = None
+        try:
+            exchange = value.get('exchange', None) or value['value']
+            if not exchange.endswith('.'):
+                reasons.append('missing trailing .')
+        except KeyError:
+            reasons.append('missing exchange')
+        return reasons
+
     def __init__(self, value):
-        # TODO: rename preference
-        self.priority = int(value['priority'])
-        # TODO: rename to exchange?
-        self.value = value['value'].lower()
+        # RFC1035 says preference, half the providers use priority
+        try:
+            preference = value['preference']
+        except KeyError:
+            preference = value['priority']
+        self.preference = int(preference)
+        # UNTIL 1.0 remove value fallback
+        try:
+            exchange = value['exchange']
+        except KeyError:
+            exchange = value['value']
+        self.exchange = exchange
 
     @property
     def data(self):
         return {
-            'priority': self.priority,
-            'value': self.value,
+            'preference': self.preference,
+            'exchange': self.exchange,
         }
 
     def __cmp__(self, other):
-        if self.priority == other.priority:
-            return cmp(self.value, other.value)
-        return cmp(self.priority, other.priority)
+        if self.preference == other.preference:
+            return cmp(self.exchange, other.exchange)
+        return cmp(self.preference, other.preference)
 
     def __repr__(self):
-        return "'{} {}'".format(self.priority, self.value)
+        return "'{} {}'".format(self.preference, self.exchange)
 
 
 class MxRecord(_ValuesMixin, Record):
     _type = 'MX'
 
+    @classmethod
+    def _validate_value(cls, value):
+        return MxValue._validate_value(value)
+
     def _process_values(self, values):
-        ret = []
-        for value in values:
-            try:
-                ret.append(MxValue(value))
-            except KeyError as e:
-                raise Exception('Invalid value in record {}, missing {}'
-                                .format(self.fqdn, e.args[0]))
-        return ret
+        return [MxValue(v) for v in values]
 
 
 class NaptrValue(object):
+    VALID_FLAGS = ('S', 'A', 'U', 'P')
+
+    @classmethod
+    def _validate_value(cls, data):
+        reasons = []
+        try:
+            int(data['order'])
+        except KeyError:
+            reasons.append('missing order')
+        except ValueError:
+            reasons.append('invalid order "{}"'.format(data['order']))
+        try:
+            int(data['preference'])
+        except KeyError:
+            reasons.append('missing preference')
+        except ValueError:
+            reasons.append('invalid preference "{}"'
+                           .format(data['preference']))
+        try:
+            flags = data['flags']
+            if flags not in cls.VALID_FLAGS:
+                reasons.append('unrecognized flags "{}"'.format(flags))
+        except KeyError:
+            reasons.append('missing flags')
+
+        # TODO: validate these... they're non-trivial
+        for k in ('service', 'regexp', 'replacement'):
+            if k not in data:
+                reasons.append('missing {}'.format(k))
+        return reasons
 
     def __init__(self, value):
         self.order = int(value['order'])
@@ -420,41 +600,70 @@ class NaptrValue(object):
 class NaptrRecord(_ValuesMixin, Record):
     _type = 'NAPTR'
 
+    @classmethod
+    def _validate_value(cls, value):
+        return NaptrValue._validate_value(value)
+
     def _process_values(self, values):
-        ret = []
-        for value in values:
-            try:
-                ret.append(NaptrValue(value))
-            except KeyError as e:
-                raise Exception('Invalid value in record {}, missing {}'
-                                .format(self.fqdn, e.args[0]))
-        return ret
+        return [NaptrValue(v) for v in values]
 
 
 class NsRecord(_ValuesMixin, Record):
     _type = 'NS'
 
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
+        if not value.endswith('.'):
+            reasons.append('missing trailing .')
+        return reasons
+
     def _process_values(self, values):
-        ret = []
-        for ns in values:
-            if not ns.endswith('.'):
-                raise Exception('Invalid record {}, value {} missing '
-                                'trailing .'.format(self.fqdn, ns))
-            ret.append(ns.lower())
-        return ret
+        return values
 
 
 class PtrRecord(_ValueMixin, Record):
     _type = 'PTR'
 
-    def _process_value(self, value):
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
         if not value.endswith('.'):
-            raise Exception('Invalid record {}, value ({}) missing trailing .'
-                            .format(self.fqdn, value))
-        return value.lower()
+            reasons.append('missing trailing .')
+        return reasons
+
+    def _process_value(self, value):
+        return value
 
 
 class SshfpValue(object):
+    VALID_ALGORITHMS = (1, 2)
+    VALID_FINGERPRINT_TYPES = (1,)
+
+    @classmethod
+    def _validate_value(cls, value):
+        reasons = []
+        try:
+            algorithm = int(value['algorithm'])
+            if algorithm not in cls.VALID_ALGORITHMS:
+                reasons.append('unrecognized algorithm "{}"'.format(algorithm))
+        except KeyError:
+            reasons.append('missing algorithm')
+        except ValueError:
+            reasons.append('invalid algorithm "{}"'.format(value['algorithm']))
+        try:
+            fingerprint_type = int(value['fingerprint_type'])
+            if fingerprint_type not in cls.VALID_FINGERPRINT_TYPES:
+                reasons.append('unrecognized fingerprint_type "{}"'
+                               .format(fingerprint_type))
+        except KeyError:
+            reasons.append('missing fingerprint_type')
+        except ValueError:
+            reasons.append('invalid fingerprint_type "{}"'
+                           .format(value['fingerprint_type']))
+        if 'fingerprint' not in value:
+            reasons.append('missing fingerprint')
+        return reasons
 
     def __init__(self, value):
         self.algorithm = int(value['algorithm'])
@@ -484,26 +693,61 @@ class SshfpValue(object):
 class SshfpRecord(_ValuesMixin, Record):
     _type = 'SSHFP'
 
+    @classmethod
+    def _validate_value(cls, value):
+        return SshfpValue._validate_value(value)
+
     def _process_values(self, values):
-        ret = []
-        for value in values:
-            try:
-                ret.append(SshfpValue(value))
-            except KeyError as e:
-                raise Exception('Invalid value in record {}, missing {}'
-                                .format(self.fqdn, e.args[0]))
-        return ret
+        return [SshfpValue(v) for v in values]
+
+
+_unescaped_semicolon_re = re.compile(r'\w;')
 
 
 class SpfRecord(_ValuesMixin, Record):
     _type = 'SPF'
 
+    @classmethod
+    def _validate_value(cls, value):
+        if _unescaped_semicolon_re.search(value):
+            return ['unescaped ;']
+        return []
+
     def _process_values(self, values):
         return values
 
 
 class SrvValue(object):
 
+    @classmethod
+    def _validate_value(self, value):
+        reasons = []
+        # TODO: validate algorithm and fingerprint_type values
+        try:
+            int(value['priority'])
+        except KeyError:
+            reasons.append('missing priority')
+        except ValueError:
+            reasons.append('invalid priority "{}"'.format(value['priority']))
+        try:
+            int(value['weight'])
+        except KeyError:
+            reasons.append('missing weight')
+        except ValueError:
+            reasons.append('invalid weight "{}"'.format(value['weight']))
+        try:
+            int(value['port'])
+        except KeyError:
+            reasons.append('missing port')
+        except ValueError:
+            reasons.append('invalid port "{}"'.format(value['port']))
+        try:
+            if not value['target'].endswith('.'):
+                reasons.append('missing trailing .')
+        except KeyError:
+            reasons.append('missing target')
+        return reasons
+
     def __init__(self, value):
         self.priority = int(value['priority'])
         self.weight = int(value['weight'])
@@ -537,28 +781,30 @@ class SrvRecord(_ValuesMixin, Record):
     _type = 'SRV'
     _name_re = re.compile(r'^_[^\.]+\.[^\.]+')
 
-    def __init__(self, zone, name, data, source=None):
-        if not self._name_re.match(name):
-            raise Exception('Invalid name {}.{}'.format(name, zone.name))
-        super(SrvRecord, self).__init__(zone, name, data, source)
+    @classmethod
+    def validate(cls, name, data):
+        reasons = []
+        if not cls._name_re.match(name):
+            reasons.append('invalid name')
+        reasons.extend(super(SrvRecord, cls).validate(name, data))
+        return reasons
+
+    @classmethod
+    def _validate_value(cls, value):
+        return SrvValue._validate_value(value)
 
     def _process_values(self, values):
-        ret = []
-        for value in values:
-            try:
-                ret.append(SrvValue(value))
-            except KeyError as e:
-                raise Exception('Invalid value in record {}, missing {}'
-                                .format(self.fqdn, e.args[0]))
-        return ret
+        return [SrvValue(v) for v in values]
 
 
 class TxtRecord(_ValuesMixin, Record):
     _type = 'TXT'
 
+    @classmethod
+    def _validate_value(cls, value):
+        if _unescaped_semicolon_re.search(value):
+            return ['unescaped ;']
+        return []
+
     def _process_values(self, values):
-        for value in values:
-            if _unescaped_semicolon_re.search(value):
-                raise Exception('Invalid record {}, unescaped ;'
-                                .format(self.fqdn))
         return values

octodns/source/base.py

@@ -16,18 +16,27 @@ class BaseSource(object):
         if not hasattr(self, 'SUPPORTS_GEO'):
             raise NotImplementedError('Abstract base class, SUPPORTS_GEO '
                                       'property missing')
+        if not hasattr(self, 'SUPPORTS'):
+            raise NotImplementedError('Abstract base class, SUPPORTS '
+                                      'property missing')
 
-    def populate(self, zone, target=False):
+    def populate(self, zone, target=False, lenient=False):
         '''
         Loads all zones the provider knows about
+
+        When `target` is True the populate call is being made to load the
+        current state of the provider.
+
+        When `lenient` is True the populate call may skip record validation and
+        do a "best effort" load of data. That will allow through some common,
+        but not best practices stuff that we otherwise would reject. E.g. no
+        trailing . or mising escapes for ;.
         '''
         raise NotImplementedError('Abstract base class, populate method '
                                   'missing')
 
     def supports(self, record):
-        # Unless overriden and handled appropriaitely we'll assume that all
-        # record types are supported
-        return True
+        return record._type in self.SUPPORTS
 
     def __repr__(self):
         return self.__class__.__name__

octodns/source/tinydns.py

@@ -19,6 +19,7 @@ from .base import BaseSource
 
 class TinyDnsBaseSource(BaseSource):
     SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'CNAME', 'MX', 'NS'))
 
     split_re = re.compile(r':+')
 
@@ -64,8 +65,8 @@ class TinyDnsBaseSource(BaseSource):
             'ttl': ttl,
             'type': _type,
             'values': [{
-                'priority': r[1],
-                'value': '{}.'.format(r[0])
+                'preference': r[1],
+                'exchange': '{}.'.format(r[0])
             } for r in records]
         }
 
@@ -80,19 +81,21 @@ class TinyDnsBaseSource(BaseSource):
             'values': ['{}.'.format(r[0]) for r in records]
         }
 
-    def populate(self, zone, target=False):
-        self.log.debug('populate: zone=%s', zone.name)
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+
         before = len(zone.records)
 
         if zone.name.endswith('in-addr.arpa.'):
-            self._populate_in_addr_arpa(zone)
+            self._populate_in_addr_arpa(zone, lenient)
         else:
-            self._populate_normal(zone)
+            self._populate_normal(zone, lenient)
 
         self.log.info('populate:   found %s records',
                       len(zone.records) - before)
 
-    def _populate_normal(self, zone):
+    def _populate_normal(self, zone, lenient):
         type_map = {
             '=': 'A',
             '^': None,
@@ -128,14 +131,15 @@ class TinyDnsBaseSource(BaseSource):
                 data_for = getattr(self, '_data_for_{}'.format(_type))
                 data = data_for(_type, d)
                 if data:
-                    record = Record.new(zone, name, data, source=self)
+                    record = Record.new(zone, name, data, source=self,
+                                        lenient=lenient)
                     try:
                         zone.add_record(record)
                     except SubzoneRecordException:
                         self.log.debug('_populate_normal: skipping subzone '
                                        'record=%s', record)
 
-    def _populate_in_addr_arpa(self, zone):
+    def _populate_in_addr_arpa(self, zone, lenient):
         name_re = re.compile('(?P<name>.+)\.{}$'.format(zone.name[:-1]))
 
         for line in self._lines():
@@ -169,7 +173,7 @@ class TinyDnsBaseSource(BaseSource):
                     'ttl': ttl,
                     'type': 'PTR',
                     'value': value
-                }, source=self)
+                }, source=self, lenient=lenient)
                 try:
                     zone.add_record(record)
                 except DuplicateRecordException:

octodns/yaml.py

@@ -5,25 +5,12 @@
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
+from natsort import natsort_keygen
 from yaml import SafeDumper, SafeLoader, load, dump
 from yaml.constructor import ConstructorError
-import re
 
 
-# zero-padded sort, simplified version of
-# https://www.xormedia.com/natural-sort-order-with-zero-padding/
-_pad_re = re.compile('\d+')
-
-
-def _zero_pad(match):
-    return '{:04d}'.format(int(match.group(0)))
-
-
-def _zero_padded_numbers(s):
-    try:
-        int(s)
-    except ValueError:
-        return _pad_re.sub(lambda d: _zero_pad(d), s)
+_natsort_key = natsort_keygen()
 
 
 # Found http://stackoverflow.com/a/21912744 which guided me on how to hook in
@@ -34,9 +21,13 @@ class SortEnforcingLoader(SafeLoader):
         self.flatten_mapping(node)
         ret = self.construct_pairs(node)
         keys = [d[0] for d in ret]
-        if keys != sorted(keys, key=_zero_padded_numbers):
-            raise ConstructorError(None, None, "keys out of order: {}"
-                                   .format(', '.join(keys)), node.start_mark)
+        keys_sorted = sorted(keys, key=_natsort_key)
+        for key in keys:
+            expected = keys_sorted.pop(0)
+            if key != expected:
+                raise ConstructorError(None, None, 'keys out of order: '
+                                       'expected {} got {} at {}'
+                                       .format(expected, key, node.start_mark))
         return dict(ret)
 
 
@@ -59,7 +50,7 @@ class SortingDumper(SafeDumper):
 
     def _representer(self, data):
         data = data.items()
-        data.sort(key=lambda d: _zero_padded_numbers(d[0]))
+        data.sort(key=lambda d: _natsort_key(d[0]))
         return self.represent_mapping(self.DEFAULT_MAPPING_TAG, data)
 
 

octodns/zone.py

@@ -5,6 +5,7 @@
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
+from collections import defaultdict
 from logging import getLogger
 import re
 
@@ -19,6 +20,10 @@ class DuplicateRecordException(Exception):
     pass
 
 
+class InvalidNodeException(Exception):
+    pass
+
+
 def _is_eligible(record):
     # Should this record be considered when computing changes
     # We ignore all top-level NS records
@@ -35,19 +40,26 @@ class Zone(object):
         # Force everyting to lowercase just to be safe
         self.name = str(name).lower() if name else name
         self.sub_zones = sub_zones
-        self.records = set()
+        # We're grouping by node, it allows us to efficently search for
+        # duplicates and detect when CNAMEs co-exist with other records
+        self._records = defaultdict(set)
         # optional leading . to match empty hostname
         # optional trailing . b/c some sources don't have it on their fqdn
         self._name_re = re.compile('\.?{}?$'.format(name))
 
         self.log.debug('__init__: zone=%s, sub_zones=%s', self, sub_zones)
 
+    @property
+    def records(self):
+        return set([r for _, node in self._records.items() for r in node])
+
     def hostname_from_fqdn(self, fqdn):
         return self._name_re.sub('', fqdn)
 
-    def add_record(self, record):
+    def add_record(self, record, replace=False):
         name = record.name
         last = name.split('.')[-1]
+
         if last in self.sub_zones:
             if name != last:
                 # it's a record for something under a sub-zone
@@ -59,10 +71,30 @@ class Zone(object):
                 raise SubzoneRecordException('Record {} a managed sub-zone '
                                              'and not of type NS'
                                              .format(record.fqdn))
-        if record in self.records:
+
+        if replace:
+            # will remove it if it exists
+            self._records[name].discard(record)
+
+        node = self._records[name]
+        if record in node:
+            # We already have a record at this node of this type
             raise DuplicateRecordException('Duplicate record {}, type {}'
-                                           .format(record.fqdn, record._type))
-        self.records.add(record)
+                                           .format(record.fqdn,
+                                                   record._type))
+        elif ((record._type == 'CNAME' and len(node) > 0) or
+              ('CNAME' in map(lambda r: r._type, node))):
+            # We're adding a CNAME to existing records or adding to an existing
+            # CNAME
+            raise InvalidNodeException('Invalid state, CNAME at {} cannot '
+                                       'coexist with other records'
+                                       .format(record.fqdn))
+
+        node.add(record)
+
+    def _remove_record(self, record):
+        'Only for use in tests'
+        self._records[record.name].discard(record)
 
     def changes(self, desired, target):
         self.log.debug('changes: zone=%s, target=%s', self, target)

requirements.txt

@@ -1,17 +1,21 @@
 # These are known good versions. You're free to use others and things will
 # likely work, but no promises are made, especilly if you go older.
 PyYaml==3.12
-boto3==1.4.4
-botocore==1.5.4
+azure-mgmt-dns==1.0.1
+azure-common==1.1.6
+boto3==1.4.6
+botocore==1.6.8
 dnspython==1.15.0
-docutils==0.13.1
-dyn==1.7.10
-futures==3.0.5
+docutils==0.14
+dyn==1.8.0
+futures==3.1.1
 incf.countryutils==1.0
 ipaddress==1.0.18
-jmespath==0.9.0
-nsone==0.9.10
-python-dateutil==2.6.0
+jmespath==0.9.3
+msrestazure==0.4.10
+natsort==5.0.3
+nsone==0.9.14
+python-dateutil==2.6.1
 requests==2.13.0
 s3transfer==0.1.10
 six==1.10.0

script/release

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+cd "$(dirname $0)"/..
+ROOT=$(pwd)
+
+VERSION=$(grep __VERSION__ $ROOT/octodns/__init__.py | sed -e "s/.* = '//" -e "s/'$//")
+
+git tag -s v$VERSION -m "Release $VERSION"
+git push origin v$VERSION
+echo "Tagged and pushed v$VERSION"
+python setup.py sdist upload
+echo "Updloaded $VERSION"

setup.py

@@ -34,6 +34,7 @@ setup(
         'futures>=3.0.5',
         'incf.countryutils>=1.0',
         'ipaddress>=1.0.18',
+        'natsort>=5.0.3',
         'python-dateutil>=2.6.0',
         'requests>=2.13.0'
     ],

tests/config/unit.tests.yaml

@@ -31,6 +31,11 @@
     values:
     - 6.2.3.4.
     - 7.2.3.4.
+  - type: CAA
+    values:
+    - flags: 0
+      tag: issue
+      value: ca.unit.tests
 _srv._tcp:
   ttl: 600
   type: SRV
@@ -60,12 +65,12 @@ mx:
   ttl: 300
   type: MX
   values:
-  - priority: 40
-    value: smtp-1.unit.tests.
-  - priority: 20
-    value: smtp-2.unit.tests.
-  - priority: 30
-    value: smtp-3.unit.tests.
+  - exchange: smtp-1.unit.tests.
+    preference: 40
+  - exchange: smtp-2.unit.tests.
+    preference: 20
+  - exchange: smtp-3.unit.tests.
+    preference: 30
   - priority: 10
     value: smtp-4.unit.tests.
 naptr:

tests/fixtures/cloudflare-dns_records-page-2.json

@@ -118,14 +118,35 @@
       "meta": {
         "auto_added": false
       }
+    },
+    {
+      "id": "fc223b34cd5611334422ab3322997667",
+      "type": "CAA",
+      "name": "unit.tests",
+      "data": {
+          "flags": 0,
+          "tag": "issue",
+          "value": "ca.unit.tests"
+      },
+      "proxiable": false,
+      "proxied": false,
+      "ttl": 3600,
+      "locked": false,
+      "zone_id": "ff12ab34cd5611334422ab3322997650",
+      "zone_name": "unit.tests",
+      "modified_on": "2017-03-11T18:01:42.961566Z",
+      "created_on": "2017-03-11T18:01:42.961566Z",
+      "meta": {
+        "auto_added": false
+      }
     }
   ],
   "result_info": {
     "page": 2,
     "per_page": 10,
     "total_pages": 2,
-    "count": 7,
-    "total_count": 17
+    "count": 8,
+    "total_count": 19
   },
   "success": true,
   "errors": [],

tests/fixtures/dnsimple-page-2.json

@@ -159,12 +159,28 @@
       "system_record": false,
       "created_at": "2017-03-09T15:55:09Z",
       "updated_at": "2017-03-09T15:55:09Z"
+    },
+    {
+      "id": 12188803,
+      "zone_id": "unit.tests",
+      "parent_id": null,
+      "name": "",
+      "content": "0 issue \"ca.unit.tests\"",
+      "ttl": 3600,
+      "priority": null,
+      "type": "CAA",
+      "regions": [
+        "global"
+      ],
+      "system_record": false,
+      "created_at": "2017-03-09T15:55:09Z",
+      "updated_at": "2017-03-09T15:55:09Z"
     }
   ],
   "pagination": {
     "current_page": 2,
     "per_page": 20,
-    "total_entries": 29,
+    "total_entries": 30,
     "total_pages": 2
   }
 }

tests/fixtures/powerdns-full-data.json

@@ -230,6 +230,18 @@
             ],
             "ttl": 300,
             "type": "A"
+        },
+        {
+            "comments": [],
+            "name": "unit.tests.",
+            "records": [
+                {
+                    "content": "0 issue \"ca.unit.tests\"",
+                    "disabled": false
+                }
+            ],
+            "ttl": 3600,
+            "type": "CAA"
         }
     ],
     "serial": 2017012803,

tests/helpers.py

@@ -17,11 +17,12 @@ class SimpleSource(object):
 
 class SimpleProvider(object):
     SUPPORTS_GEO = False
+    SUPPORTS = set(('A',))
 
     def __init__(self, id='test'):
         pass
 
-    def populate(self, zone, source=True):
+    def populate(self, zone, source=False, lenient=False):
         pass
 
     def supports(self, record):
@@ -37,7 +38,7 @@ class GeoProvider(object):
     def __init__(self, id='test'):
         pass
 
-    def populate(self, zone, source=True):
+    def populate(self, zone, source=False, lenient=False):
         pass
 
     def supports(self, record):

tests/test_octodns_manager.py

@@ -128,6 +128,12 @@ class TestManager(TestCase):
                 .sync(dry_run=False, force=True)
             self.assertEquals(19, tc)
 
+            # Include meta
+            tc = Manager(get_config_filename('simple.yaml'), max_workers=1,
+                         include_meta=True) \
+                .sync(dry_run=False, force=True)
+            self.assertEquals(23, tc)
+
     def test_eligible_targets(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
@@ -195,15 +201,15 @@ class TestManager(TestCase):
             manager = Manager(get_config_filename('simple.yaml'))
 
             with self.assertRaises(Exception) as ctx:
-                manager.dump('unit.tests.', tmpdir.dirname, 'nope')
+                manager.dump('unit.tests.', tmpdir.dirname, False, 'nope')
             self.assertEquals('Unknown source: nope', ctx.exception.message)
 
-            manager.dump('unit.tests.', tmpdir.dirname, 'in')
+            manager.dump('unit.tests.', tmpdir.dirname, False, 'in')
 
             # make sure this fails with an IOError and not a KeyError when
             # tyring to find sub zones
             with self.assertRaises(IOError):
-                manager.dump('unknown.zone.', tmpdir.dirname, 'in')
+                manager.dump('unknown.zone.', tmpdir.dirname, False, 'in')
 
     def test_validate_configs(self):
         Manager(get_config_filename('simple-validate.yaml')).validate_configs()

tests/test_octodns_provider_azuredns.py

@@ -0,0 +1,379 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from octodns.record import Create, Delete, Record
+from octodns.provider.azuredns import _AzureRecord, AzureProvider, \
+    _check_endswith_dot, _parse_azure_type
+from octodns.zone import Zone
+from octodns.provider.base import Plan
+
+from azure.mgmt.dns.models import ARecord, AaaaRecord, CnameRecord, MxRecord, \
+    SrvRecord, NsRecord, PtrRecord, TxtRecord, RecordSet, SoaRecord, \
+    Zone as AzureZone
+from msrestazure.azure_exceptions import CloudError
+
+from unittest import TestCase
+from mock import Mock, patch
+
+
+zone = Zone(name='unit.tests.', sub_zones=[])
+octo_records = []
+octo_records.append(Record.new(zone, '', {
+    'ttl': 0,
+    'type': 'A',
+    'values': ['1.2.3.4', '10.10.10.10']}))
+octo_records.append(Record.new(zone, 'a', {
+    'ttl': 1,
+    'type': 'A',
+    'values': ['1.2.3.4', '1.1.1.1']}))
+octo_records.append(Record.new(zone, 'aa', {
+    'ttl': 9001,
+    'type': 'A',
+    'values': ['1.2.4.3']}))
+octo_records.append(Record.new(zone, 'aaa', {
+    'ttl': 2,
+    'type': 'A',
+    'values': ['1.1.1.3']}))
+octo_records.append(Record.new(zone, 'cname', {
+    'ttl': 3,
+    'type': 'CNAME',
+    'value': 'a.unit.tests.'}))
+octo_records.append(Record.new(zone, 'mx1', {
+    'ttl': 3,
+    'type': 'MX',
+    'values': [{
+        'priority': 10,
+        'value': 'mx1.unit.tests.',
+    }, {
+        'priority': 20,
+        'value': 'mx2.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, 'mx2', {
+    'ttl': 3,
+    'type': 'MX',
+    'values': [{
+        'priority': 10,
+        'value': 'mx1.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, '', {
+    'ttl': 4,
+    'type': 'NS',
+    'values': ['ns1.unit.tests.', 'ns2.unit.tests.']}))
+octo_records.append(Record.new(zone, 'foo', {
+    'ttl': 5,
+    'type': 'NS',
+    'value': 'ns1.unit.tests.'}))
+octo_records.append(Record.new(zone, '_srv._tcp', {
+    'ttl': 6,
+    'type': 'SRV',
+    'values': [{
+        'priority': 10,
+        'weight': 20,
+        'port': 30,
+        'target': 'foo-1.unit.tests.',
+    }, {
+        'priority': 12,
+        'weight': 30,
+        'port': 30,
+        'target': 'foo-2.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, '_srv2._tcp', {
+    'ttl': 7,
+    'type': 'SRV',
+    'values': [{
+        'priority': 12,
+        'weight': 17,
+        'port': 1,
+        'target': 'srvfoo.unit.tests.',
+    }]}))
+octo_records.append(Record.new(zone, 'txt1', {
+    'ttl': 8,
+    'type': 'TXT',
+    'value': 'txt singleton test'}))
+octo_records.append(Record.new(zone, 'txt2', {
+    'ttl': 9,
+    'type': 'TXT',
+    'values': ['txt multiple test', 'txt multiple test 2']}))
+
+azure_records = []
+_base0 = _AzureRecord('TestAzure', octo_records[0])
+_base0.zone_name = 'unit.tests'
+_base0.relative_record_set_name = '@'
+_base0.record_type = 'A'
+_base0.params['ttl'] = 0
+_base0.params['arecords'] = [ARecord('1.2.3.4'), ARecord('10.10.10.10')]
+azure_records.append(_base0)
+
+_base1 = _AzureRecord('TestAzure', octo_records[1])
+_base1.zone_name = 'unit.tests'
+_base1.relative_record_set_name = 'a'
+_base1.record_type = 'A'
+_base1.params['ttl'] = 1
+_base1.params['arecords'] = [ARecord('1.2.3.4'), ARecord('1.1.1.1')]
+azure_records.append(_base1)
+
+_base2 = _AzureRecord('TestAzure', octo_records[2])
+_base2.zone_name = 'unit.tests'
+_base2.relative_record_set_name = 'aa'
+_base2.record_type = 'A'
+_base2.params['ttl'] = 9001
+_base2.params['arecords'] = ARecord('1.2.4.3')
+azure_records.append(_base2)
+
+_base3 = _AzureRecord('TestAzure', octo_records[3])
+_base3.zone_name = 'unit.tests'
+_base3.relative_record_set_name = 'aaa'
+_base3.record_type = 'A'
+_base3.params['ttl'] = 2
+_base3.params['arecords'] = ARecord('1.1.1.3')
+azure_records.append(_base3)
+
+_base4 = _AzureRecord('TestAzure', octo_records[4])
+_base4.zone_name = 'unit.tests'
+_base4.relative_record_set_name = 'cname'
+_base4.record_type = 'CNAME'
+_base4.params['ttl'] = 3
+_base4.params['cname_record'] = CnameRecord('a.unit.tests.')
+azure_records.append(_base4)
+
+_base5 = _AzureRecord('TestAzure', octo_records[5])
+_base5.zone_name = 'unit.tests'
+_base5.relative_record_set_name = 'mx1'
+_base5.record_type = 'MX'
+_base5.params['ttl'] = 3
+_base5.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.'),
+                               MxRecord(20, 'mx2.unit.tests.')]
+azure_records.append(_base5)
+
+_base6 = _AzureRecord('TestAzure', octo_records[6])
+_base6.zone_name = 'unit.tests'
+_base6.relative_record_set_name = 'mx2'
+_base6.record_type = 'MX'
+_base6.params['ttl'] = 3
+_base6.params['mx_records'] = [MxRecord(10, 'mx1.unit.tests.')]
+azure_records.append(_base6)
+
+_base7 = _AzureRecord('TestAzure', octo_records[7])
+_base7.zone_name = 'unit.tests'
+_base7.relative_record_set_name = '@'
+_base7.record_type = 'NS'
+_base7.params['ttl'] = 4
+_base7.params['ns_records'] = [NsRecord('ns1.unit.tests.'),
+                               NsRecord('ns2.unit.tests.')]
+azure_records.append(_base7)
+
+_base8 = _AzureRecord('TestAzure', octo_records[8])
+_base8.zone_name = 'unit.tests'
+_base8.relative_record_set_name = 'foo'
+_base8.record_type = 'NS'
+_base8.params['ttl'] = 5
+_base8.params['ns_records'] = [NsRecord('ns1.unit.tests.')]
+azure_records.append(_base8)
+
+_base9 = _AzureRecord('TestAzure', octo_records[9])
+_base9.zone_name = 'unit.tests'
+_base9.relative_record_set_name = '_srv._tcp'
+_base9.record_type = 'SRV'
+_base9.params['ttl'] = 6
+_base9.params['srv_records'] = [SrvRecord(10, 20, 30, 'foo-1.unit.tests.'),
+                                SrvRecord(12, 30, 30, 'foo-2.unit.tests.')]
+azure_records.append(_base9)
+
+_base10 = _AzureRecord('TestAzure', octo_records[10])
+_base10.zone_name = 'unit.tests'
+_base10.relative_record_set_name = '_srv2._tcp'
+_base10.record_type = 'SRV'
+_base10.params['ttl'] = 7
+_base10.params['srv_records'] = [SrvRecord(12, 17, 1, 'srvfoo.unit.tests.')]
+azure_records.append(_base10)
+
+_base11 = _AzureRecord('TestAzure', octo_records[11])
+_base11.zone_name = 'unit.tests'
+_base11.relative_record_set_name = 'txt1'
+_base11.record_type = 'TXT'
+_base11.params['ttl'] = 8
+_base11.params['txt_records'] = [TxtRecord(['txt singleton test'])]
+azure_records.append(_base11)
+
+_base12 = _AzureRecord('TestAzure', octo_records[12])
+_base12.zone_name = 'unit.tests'
+_base12.relative_record_set_name = 'txt2'
+_base12.record_type = 'TXT'
+_base12.params['ttl'] = 9
+_base12.params['txt_records'] = [TxtRecord(['txt multiple test']),
+                                 TxtRecord(['txt multiple test 2'])]
+azure_records.append(_base12)
+
+
+class Test_AzureRecord(TestCase):
+    def test_azure_record(self):
+        assert(len(azure_records) == len(octo_records))
+        for i in range(len(azure_records)):
+            octo = _AzureRecord('TestAzure', octo_records[i])
+            assert(azure_records[i]._equals(octo))
+
+
+class Test_ParseAzureType(TestCase):
+    def test_parse_azure_type(self):
+        for expected, test in [['A', 'Microsoft.Network/dnszones/A'],
+                               ['AAAA', 'Microsoft.Network/dnszones/AAAA'],
+                               ['NS', 'Microsoft.Network/dnszones/NS'],
+                               ['MX', 'Microsoft.Network/dnszones/MX']]:
+                self.assertEquals(expected, _parse_azure_type(test))
+
+
+class Test_CheckEndswithDot(TestCase):
+    def test_check_endswith_dot(self):
+        for expected, test in [['a.', 'a'],
+                               ['a.', 'a.'],
+                               ['foo.bar.', 'foo.bar.'],
+                               ['foo.bar.', 'foo.bar']]:
+            self.assertEquals(expected, _check_endswith_dot(test))
+
+
+class TestAzureDnsProvider(TestCase):
+    def _provider(self):
+        return self._get_provider('mock_spc', 'mock_dns_client')
+
+    @patch('octodns.provider.azuredns.DnsManagementClient')
+    @patch('octodns.provider.azuredns.ServicePrincipalCredentials')
+    def _get_provider(self, mock_spc, mock_dns_client):
+        '''Returns a mock AzureProvider object to use in testing.
+
+            :param mock_spc: placeholder
+            :type  mock_spc: str
+            :param mock_dns_client: placeholder
+            :type  mock_dns_client: str
+
+            :type return: AzureProvider
+        '''
+        return AzureProvider('mock_id', 'mock_client', 'mock_key',
+                             'mock_directory', 'mock_sub', 'mock_rg')
+
+    def test_populate_records(self):
+        provider = self._get_provider()
+
+        rs = []
+        rs.append(RecordSet(name='a1', ttl=0, type='A',
+                            arecords=[ARecord('1.1.1.1')]))
+        rs.append(RecordSet(name='a2', ttl=1, type='A',
+                            arecords=[ARecord('1.1.1.1'),
+                                      ARecord('2.2.2.2')]))
+        rs.append(RecordSet(name='aaaa1', ttl=2, type='AAAA',
+                            aaaa_records=[AaaaRecord('1:1ec:1::1')]))
+        rs.append(RecordSet(name='aaaa2', ttl=3, type='AAAA',
+                            aaaa_records=[AaaaRecord('1:1ec:1::1'),
+                                          AaaaRecord('1:1ec:1::2')]))
+        rs.append(RecordSet(name='cname1', ttl=4, type='CNAME',
+                            cname_record=CnameRecord('cname.unit.test.')))
+        rs.append(RecordSet(name='cname2', ttl=5, type='CNAME',
+                            cname_record=None))
+        rs.append(RecordSet(name='mx1', ttl=6, type='MX',
+                            mx_records=[MxRecord(10, 'mx1.unit.test.')]))
+        rs.append(RecordSet(name='mx2', ttl=7, type='MX',
+                            mx_records=[MxRecord(10, 'mx1.unit.test.'),
+                                        MxRecord(11, 'mx2.unit.test.')]))
+        rs.append(RecordSet(name='ns1', ttl=8, type='NS',
+                            ns_records=[NsRecord('ns1.unit.test.')]))
+        rs.append(RecordSet(name='ns2', ttl=9, type='NS',
+                            ns_records=[NsRecord('ns1.unit.test.'),
+                                        NsRecord('ns2.unit.test.')]))
+        rs.append(RecordSet(name='ptr1', ttl=10, type='PTR',
+                            ptr_records=[PtrRecord('ptr1.unit.test.')]))
+        rs.append(RecordSet(name='ptr2', ttl=11, type='PTR',
+                            ptr_records=[PtrRecord(None)]))
+        rs.append(RecordSet(name='_srv1._tcp', ttl=12, type='SRV',
+                            srv_records=[SrvRecord(1, 2, 3, '1unit.tests.')]))
+        rs.append(RecordSet(name='_srv2._tcp', ttl=13, type='SRV',
+                            srv_records=[SrvRecord(1, 2, 3, '1unit.tests.'),
+                                         SrvRecord(4, 5, 6, '2unit.tests.')]))
+        rs.append(RecordSet(name='txt1', ttl=14, type='TXT',
+                            txt_records=[TxtRecord('sample text1')]))
+        rs.append(RecordSet(name='txt2', ttl=15, type='TXT',
+                            txt_records=[TxtRecord('sample text1'),
+                                         TxtRecord('sample text2')]))
+        rs.append(RecordSet(name='', ttl=16, type='SOA',
+                            soa_record=[SoaRecord()]))
+
+        record_list = provider._dns_client.record_sets.list_by_dns_zone
+        record_list.return_value = rs
+
+        provider.populate(zone)
+
+        self.assertEquals(len(zone.records), 16)
+
+    def test_populate_zone(self):
+        provider = self._get_provider()
+
+        zone_list = provider._dns_client.zones.list_by_resource_group
+        zone_list.return_value = [AzureZone(location='global'),
+                                  AzureZone(location='global')]
+
+        provider._populate_zones()
+
+        self.assertEquals(len(provider._azure_zones), 1)
+
+    def test_bad_zone_response(self):
+        provider = self._get_provider()
+
+        _get = provider._dns_client.zones.get
+        _get.side_effect = CloudError(Mock(status=404), 'Azure Error')
+        trip = False
+        try:
+            provider._check_zone('unit.test', create=False)
+        except CloudError:
+            trip = True
+        self.assertEquals(trip, True)
+
+    def test_apply(self):
+        provider = self._get_provider()
+
+        changes = []
+        deletes = []
+        for i in octo_records:
+            changes.append(Create(i))
+            deletes.append(Delete(i))
+
+        self.assertEquals(13, provider.apply(Plan(None, zone, changes)))
+        self.assertEquals(13, provider.apply(Plan(zone, zone, deletes)))
+
+    def test_create_zone(self):
+        provider = self._get_provider()
+
+        changes = []
+        for i in octo_records:
+            changes.append(Create(i))
+        desired = Zone('unit2.test.', [])
+
+        err_msg = 'The Resource \'Microsoft.Network/dnszones/unit2.test\' '
+        err_msg += 'under resource group \'mock_rg\' was not found.'
+        _get = provider._dns_client.zones.get
+        _get.side_effect = CloudError(Mock(status=404), err_msg)
+
+        self.assertEquals(13, provider.apply(Plan(None, desired, changes)))
+
+    def test_check_zone_no_create(self):
+        provider = self._get_provider()
+
+        rs = []
+        rs.append(RecordSet(name='a1', ttl=0, type='A',
+                            arecords=[ARecord('1.1.1.1')]))
+        rs.append(RecordSet(name='a2', ttl=1, type='A',
+                            arecords=[ARecord('1.1.1.1'),
+                                      ARecord('2.2.2.2')]))
+
+        record_list = provider._dns_client.record_sets.list_by_dns_zone
+        record_list.return_value = rs
+
+        err_msg = 'The Resource \'Microsoft.Network/dnszones/unit3.test\' '
+        err_msg += 'under resource group \'mock_rg\' was not found.'
+        _get = provider._dns_client.zones.get
+        _get.side_effect = CloudError(Mock(status=404), err_msg)
+
+        provider.populate(Zone('unit3.test.', []))
+
+        self.assertEquals(len(zone.records), 0)

tests/test_octodns_provider_base.py

@@ -16,13 +16,15 @@ from octodns.zone import Zone
 class HelperProvider(BaseProvider):
     log = getLogger('HelperProvider')
 
+    SUPPORTS = set(('A',))
+
     def __init__(self, extra_changes, apply_disabled=False,
                  include_change_callback=None):
         self.__extra_changes = extra_changes
         self.apply_disabled = apply_disabled
         self.include_change_callback = include_change_callback
 
-    def populate(self, zone, target=False):
+    def populate(self, zone, target=False, lenient=False):
         pass
 
     def _include_change(self, change):
@@ -58,12 +60,19 @@ class TestBaseProvider(TestCase):
         zone = Zone('unit.tests.', [])
         with self.assertRaises(NotImplementedError) as ctx:
             HasSupportsGeo('hassupportesgeo').populate(zone)
+        self.assertEquals('Abstract base class, SUPPORTS property missing',
+                          ctx.exception.message)
+
+        class HasSupports(HasSupportsGeo):
+            SUPPORTS = set(('A',))
+        with self.assertRaises(NotImplementedError) as ctx:
+            HasSupports('hassupportes').populate(zone)
         self.assertEquals('Abstract base class, populate method missing',
                           ctx.exception.message)
 
-        class HasPopulate(HasSupportsGeo):
+        class HasPopulate(HasSupports):
 
-            def populate(self, zone, target=False):
+            def populate(self, zone, target=False, lenient=False):
                 zone.add_record(Record.new(zone, '', {
                     'ttl': 60,
                     'type': 'A',
@@ -81,7 +90,7 @@ class TestBaseProvider(TestCase):
             'value': '1.2.3.4'
         }))
 
-        self.assertTrue(HasSupportsGeo('hassupportesgeo')
+        self.assertTrue(HasSupports('hassupportesgeo')
                         .supports(list(zone.records)[0]))
 
         plan = HasPopulate('haspopulate').plan(zone)
@@ -205,9 +214,11 @@ class TestBaseProvider(TestCase):
                    for i in range(int(Plan.MIN_EXISTING_RECORDS *
                                       Plan.MAX_SAFE_UPDATE_PCENT) + 1)]
 
-        with self.assertRaises(UnsafePlan):
+        with self.assertRaises(UnsafePlan) as ctx:
             Plan(zone, zone, changes).raise_if_unsafe()
 
+        self.assertTrue('Too many updates' in ctx.exception.message)
+
     def test_safe_updates_min_existing_pcent(self):
         # MAX_SAFE_UPDATE_PCENT is safe when more
         # than MIN_EXISTING_RECORDS exist
@@ -251,9 +262,11 @@ class TestBaseProvider(TestCase):
                    for i in range(int(Plan.MIN_EXISTING_RECORDS *
                                       Plan.MAX_SAFE_DELETE_PCENT) + 1)]
 
-        with self.assertRaises(UnsafePlan):
+        with self.assertRaises(UnsafePlan) as ctx:
             Plan(zone, zone, changes).raise_if_unsafe()
 
+        self.assertTrue('Too many deletes' in ctx.exception.message)
+
     def test_safe_deletes_min_existing_pcent(self):
         # MAX_SAFE_DELETE_PCENT is safe when more
         # than MIN_EXISTING_RECORDS exist

tests/test_octodns_provider_cloudflare.py

@@ -33,7 +33,7 @@ class TestCloudflareProvider(TestCase):
     }))
     for record in list(expected.records):
         if record.name == 'sub' and record._type == 'NS':
-            expected.records.remove(record)
+            expected._remove_record(record)
             break
 
     empty = {'result': [], 'result_info': {'count': 0, 'per_page': 0}}
@@ -118,7 +118,7 @@ class TestCloudflareProvider(TestCase):
 
             zone = Zone('unit.tests.', [])
             provider.populate(zone)
-            self.assertEquals(9, len(zone.records))
+            self.assertEquals(10, len(zone.records))
 
             changes = self.expected.changes(zone, provider)
             self.assertEquals(0, len(changes))
@@ -126,7 +126,7 @@ class TestCloudflareProvider(TestCase):
         # re-populating the same zone/records comes out of cache, no calls
         again = Zone('unit.tests.', [])
         provider.populate(again)
-        self.assertEquals(9, len(again.records))
+        self.assertEquals(10, len(again.records))
 
     def test_apply(self):
         provider = CloudflareProvider('test', 'email', 'token')
@@ -140,12 +140,12 @@ class TestCloudflareProvider(TestCase):
                     'id': 42,
                 }
             },  # zone create
-        ] + [None] * 16  # individual record creates
+        ] + [None] * 17  # individual record creates
 
         # non-existant zone, create everything
         plan = provider.plan(self.expected)
-        self.assertEquals(9, len(plan.changes))
-        self.assertEquals(9, provider.apply(plan))
+        self.assertEquals(10, len(plan.changes))
+        self.assertEquals(10, provider.apply(plan))
 
         provider._request.assert_has_calls([
             # created the domain
@@ -170,7 +170,7 @@ class TestCloudflareProvider(TestCase):
             }),
         ], True)
         # expected number of total calls
-        self.assertEquals(18, provider._request.call_count)
+        self.assertEquals(19, provider._request.call_count)
 
         provider._request.reset_mock()
 

tests/test_octodns_provider_dnsimple.py

@@ -33,7 +33,7 @@ class TestDnsimpleProvider(TestCase):
     }))
     for record in list(expected.records):
         if record.name == 'sub' and record._type == 'NS':
-            expected.records.remove(record)
+            expected._remove_record(record)
             break
 
     def test_populate(self):
@@ -78,14 +78,14 @@ class TestDnsimpleProvider(TestCase):
 
             zone = Zone('unit.tests.', [])
             provider.populate(zone)
-            self.assertEquals(14, len(zone.records))
+            self.assertEquals(15, len(zone.records))
             changes = self.expected.changes(zone, provider)
             self.assertEquals(0, len(changes))
 
         # 2nd populate makes no network calls/all from cache
         again = Zone('unit.tests.', [])
         provider.populate(again)
-        self.assertEquals(14, len(again.records))
+        self.assertEquals(15, len(again.records))
 
         # bust the cache
         del provider._zone_records[zone.name]
@@ -147,7 +147,7 @@ class TestDnsimpleProvider(TestCase):
             }),
         ])
         # expected number of total calls
-        self.assertEquals(26, provider._client._request.call_count)
+        self.assertEquals(27, provider._client._request.call_count)
 
         provider._client._request.reset_mock()
 

tests/test_octodns_provider_dyn.py

@@ -46,11 +46,11 @@ class TestDynProvider(TestCase):
             'type': 'MX',
             'ttl': 302,
             'values': [{
-                'priority': 10,
-                'value': 'smtp-1.unit.tests.'
+                'preference': 10,
+                'exchange': 'smtp-1.unit.tests.'
             }, {
-                'priority': 20,
-                'value': 'smtp-2.unit.tests.'
+                'preference': 20,
+                'exchange': 'smtp-2.unit.tests.'
             }]
         }),
         ('naptr', {
@@ -109,6 +109,14 @@ class TestDynProvider(TestCase):
                 'weight': 22,
                 'port': 20,
                 'target': 'foo-2.unit.tests.'
+            }]}),
+        ('', {
+            'type': 'CAA',
+            'ttl': 308,
+            'values': [{
+                'flags': 0,
+                'tag': 'issue',
+                'value': 'ca.unit.tests'
             }]})):
         expected.add_record(Record.new(expected, name, data))
 
@@ -321,6 +329,16 @@ class TestDynProvider(TestCase):
                     'ttl': 307,
                     'zone': 'unit.tests',
                 }],
+                'caa_records': [{
+                    'fqdn': 'unit.tests',
+                    'rdata': {'flags': 0,
+                              'tag': 'issue',
+                              'value': 'ca.unit.tests'},
+                    'record_id': 12,
+                    'record_type': 'cAA',
+                    'ttl': 308,
+                    'zone': 'unit.tests',
+                }],
             }}
         ]
         got = Zone('unit.tests.', [])
@@ -414,10 +432,10 @@ class TestDynProvider(TestCase):
                 update_mock.assert_called()
             add_mock.assert_called()
             # Once for each dyn record (8 Records, 2 of which have dual values)
-            self.assertEquals(14, len(add_mock.call_args_list))
+            self.assertEquals(15, len(add_mock.call_args_list))
         execute_mock.assert_has_calls([call('/Zone/unit.tests/', 'GET', {}),
                                        call('/Zone/unit.tests/', 'GET', {})])
-        self.assertEquals(9, len(plan.changes))
+        self.assertEquals(10, len(plan.changes))
 
         execute_mock.reset_mock()
 

tests/test_octodns_provider_ns1.py

@@ -6,7 +6,8 @@ from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
 from mock import Mock, call, patch
-from nsone.rest.errors import AuthException, ResourceException
+from nsone.rest.errors import AuthException, RateLimitException, \
+    ResourceException
 from unittest import TestCase
 
 from octodns.record import Delete, Record, Update
@@ -44,11 +45,11 @@ class TestNs1Provider(TestCase):
         'ttl': 35,
         'type': 'MX',
         'values': [{
-            'priority': 10,
-            'value': 'mx1.unit.tests.',
+            'preference': 10,
+            'exchange': 'mx1.unit.tests.',
         }, {
-            'priority': 20,
-            'value': 'mx2.unit.tests.',
+            'preference': 20,
+            'exchange': 'mx2.unit.tests.',
         }]
     }))
     expected.add(Record.new(zone, 'naptr', {
@@ -95,6 +96,15 @@ class TestNs1Provider(TestCase):
         'type': 'NS',
         'values': ['ns3.unit.tests.', 'ns4.unit.tests.'],
     }))
+    expected.add(Record.new(zone, '', {
+        'ttl': 40,
+        'type': 'CAA',
+        'value': {
+            'flags': 0,
+            'tag': 'issue',
+            'value': 'ca.unit.tests',
+        },
+    }))
 
     nsone_records = [{
         'type': 'A',
@@ -140,6 +150,11 @@ class TestNs1Provider(TestCase):
         'ttl': 39,
         'short_answers': ['ns3.unit.tests.', 'ns4.unit.tests.'],
         'domain': 'sub.unit.tests.',
+    }, {
+        'type': 'CAA',
+        'ttl': 40,
+        'short_answers': ['0 issue ca.unit.tests'],
+        'domain': 'unit.tests.',
     }]
 
     @patch('nsone.NSONE.loadZone')
@@ -195,7 +210,8 @@ class TestNs1Provider(TestCase):
         provider = Ns1Provider('test', 'api-key')
 
         desired = Zone('unit.tests.', [])
-        desired.records.update(self.expected)
+        for r in self.expected:
+            desired.add_record(r)
 
         plan = provider.plan(desired)
         # everything except the root NS
@@ -225,7 +241,15 @@ class TestNs1Provider(TestCase):
         create_mock.reset_mock()
         load_mock.side_effect = \
             ResourceException('server error: zone not found')
-        create_mock.side_effect = None
+        # ugh, need a mock zone with a mock prop since we're using getattr, we
+        # can actually control side effects on `meth` with that.
+        mock_zone = Mock()
+        mock_zone.add_SRV = Mock()
+        mock_zone.add_SRV.side_effect = [
+            RateLimitException('boo', period=0),
+            None,
+        ]
+        create_mock.side_effect = [mock_zone]
         got_n = provider.apply(plan)
         self.assertEquals(expected_n, got_n)
 
@@ -245,12 +269,60 @@ class TestNs1Provider(TestCase):
         self.assertEquals(2, len(plan.changes))
         self.assertIsInstance(plan.changes[0], Update)
         self.assertIsInstance(plan.changes[1], Delete)
-
+        # ugh, we need a mock record that can be returned from loadRecord for
+        # the update and delete targets, we can add our side effects to that to
+        # trigger rate limit handling
+        mock_record = Mock()
+        mock_record.update.side_effect = [
+            RateLimitException('one', period=0),
+            None,
+        ]
+        mock_record.delete.side_effect = [
+            RateLimitException('two', period=0),
+            None,
+        ]
+        nsone_zone.loadRecord.side_effect = [mock_record, mock_record]
         got_n = provider.apply(plan)
         self.assertEquals(2, got_n)
         nsone_zone.loadRecord.assert_has_calls([
             call('unit.tests', u'A'),
-            call().update(answers=[u'1.2.3.4'], ttl=32),
             call('delete-me', u'A'),
-            call().delete()
         ])
+        mock_record.assert_has_calls([
+            call.update(answers=[u'1.2.3.4'], ttl=32),
+            call.delete()
+        ])
+
+    def test_escaping(self):
+        provider = Ns1Provider('test', 'api-key')
+
+        record = {
+            'ttl': 31,
+            'short_answers': ['foo; bar baz; blip']
+        }
+        self.assertEquals(['foo\; bar baz\; blip'],
+                          provider._data_for_SPF('SPF', record)['values'])
+
+        record = {
+            'ttl': 31,
+            'short_answers': ['no', 'foo; bar baz; blip', 'yes']
+        }
+        self.assertEquals(['no', 'foo\; bar baz\; blip', 'yes'],
+                          provider._data_for_TXT('TXT', record)['values'])
+
+        zone = Zone('unit.tests.', [])
+        record = Record.new(zone, 'spf', {
+            'ttl': 34,
+            'type': 'SPF',
+            'value': 'foo\; bar baz\; blip'
+        })
+        self.assertEquals(['foo; bar baz; blip'],
+                          provider._params_for_SPF(record)['answers'])
+
+        record = Record.new(zone, 'txt', {
+            'ttl': 35,
+            'type': 'TXT',
+            'value': 'foo\; bar baz\; blip'
+        })
+        self.assertEquals(['foo; bar baz; blip'],
+                          provider._params_for_TXT(record)['answers'])

tests/test_octodns_provider_powerdns.py

@@ -79,7 +79,7 @@ class TestPowerDnsProvider(TestCase):
         source = YamlProvider('test', join(dirname(__file__), 'config'))
         source.populate(expected)
         expected_n = len(expected.records) - 1
-        self.assertEquals(14, expected_n)
+        self.assertEquals(15, expected_n)
 
         # No diffs == no changes
         with requests_mock() as mock:
@@ -87,7 +87,7 @@ class TestPowerDnsProvider(TestCase):
 
             zone = Zone('unit.tests.', [])
             provider.populate(zone)
-            self.assertEquals(14, len(zone.records))
+            self.assertEquals(15, len(zone.records))
             changes = expected.changes(zone, provider)
             self.assertEquals(0, len(changes))
 
@@ -167,7 +167,7 @@ class TestPowerDnsProvider(TestCase):
         expected = Zone('unit.tests.', [])
         source = YamlProvider('test', join(dirname(__file__), 'config'))
         source.populate(expected)
-        self.assertEquals(15, len(expected.records))
+        self.assertEquals(16, len(expected.records))
 
         # A small change to a single record
         with requests_mock() as mock:
@@ -253,7 +253,7 @@ class TestPowerDnsProvider(TestCase):
             plan = provider.plan(expected)
             self.assertFalse(plan)
             # remove it now that we don't need the unrelated change any longer
-            expected.records.remove(unrelated_record)
+            expected._remove_record(unrelated_record)
 
         # ttl diff
         with requests_mock() as mock:

tests/test_octodns_provider_route53.py

@@ -11,8 +11,8 @@ from unittest import TestCase
 from mock import patch
 
 from octodns.record import Create, Delete, Record, Update
-from octodns.provider.route53 import _Route53Record, Route53Provider, \
-    _octal_replace
+from octodns.provider.route53 import Route53Provider, _Route53GeoDefault, \
+    _Route53GeoRecord, _Route53Record, _octal_replace
 from octodns.zone import Zone
 
 from helpers import GeoProvider
@@ -52,11 +52,11 @@ class TestRoute53Provider(TestCase):
                                                       'Goodbye World?']}),
         ('', {'ttl': 64, 'type': 'MX',
               'values': [{
-                  'priority': 10,
-                  'value': 'smtp-1.unit.tests.',
+                  'preference': 10,
+                  'exchange': 'smtp-1.unit.tests.',
               }, {
-                  'priority': 20,
-                  'value': 'smtp-2.unit.tests.',
+                  'preference': 20,
+                  'exchange': 'smtp-2.unit.tests.',
               }]}),
         ('naptr', {'ttl': 65, 'type': 'NAPTR',
                    'value': {
@@ -77,6 +77,12 @@ class TestRoute53Provider(TestCase):
          {'ttl': 67, 'type': 'NS', 'values': ['8.2.3.4.', '9.2.3.4.']}),
         ('sub',
          {'ttl': 68, 'type': 'NS', 'values': ['5.2.3.4.', '6.2.3.4.']}),
+        ('',
+         {'ttl': 69, 'type': 'CAA', 'value': {
+             'flags': 0,
+             'tag': 'issue',
+             'value': 'ca.unit.tests'
+         }}),
     ):
         record = Record.new(expected, name, data)
         expected.add_record(record)
@@ -300,6 +306,13 @@ class TestRoute53Provider(TestCase):
                     'Value': 'ns1.unit.tests.',
                 }],
                 'TTL': 69,
+            }, {
+                'Name': 'unit.tests.',
+                'Type': 'CAA',
+                'ResourceRecords': [{
+                    'Value': '0 issue "ca.unit.tests"',
+                }],
+                'TTL': 69,
             }],
             'IsTruncated': False,
             'MaxItems': '100',
@@ -347,7 +360,7 @@ class TestRoute53Provider(TestCase):
                              {'HostedZoneId': 'z42'})
 
         plan = provider.plan(self.expected)
-        self.assertEquals(8, len(plan.changes))
+        self.assertEquals(9, len(plan.changes))
         for change in plan.changes:
             self.assertIsInstance(change, Create)
         stubber.assert_no_pending_responses()
@@ -366,17 +379,17 @@ class TestRoute53Provider(TestCase):
                                  'SubmittedAt': '2017-01-29T01:02:03Z',
                              }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY})
 
-        self.assertEquals(8, provider.apply(plan))
+        self.assertEquals(9, provider.apply(plan))
         stubber.assert_no_pending_responses()
 
         # Delete by monkey patching in a populate that includes an extra record
-        def add_extra_populate(existing, target):
+        def add_extra_populate(existing, target, lenient):
             for record in self.expected.records:
-                existing.records.add(record)
+                existing.add_record(record)
             record = Record.new(existing, 'extra',
                                 {'ttl': 99, 'type': 'A',
                                  'values': ['9.9.9.9']})
-            existing.records.add(record)
+            existing.add_record(record)
 
         provider.populate = add_extra_populate
         change_resource_record_sets_params = {
@@ -406,10 +419,10 @@ class TestRoute53Provider(TestCase):
 
         # Update by monkey patching in a populate that modifies the A record
         # with geos
-        def mod_geo_populate(existing, target):
+        def mod_geo_populate(existing, target, lenient):
             for record in self.expected.records:
                 if record._type != 'A' or not record.geo:
-                    existing.records.add(record)
+                    existing.add_record(record)
             record = Record.new(existing, '', {
                 'ttl': 61,
                 'type': 'A',
@@ -420,7 +433,7 @@ class TestRoute53Provider(TestCase):
                     'NA-US-KY': ['7.2.3.4']
                 }
             })
-            existing.records.add(record)
+            existing.add_record(record)
 
         provider.populate = mod_geo_populate
         change_resource_record_sets_params = {
@@ -502,10 +515,10 @@ class TestRoute53Provider(TestCase):
 
         # Update converting to non-geo by monkey patching in a populate that
         # modifies the A record with geos
-        def mod_add_geo_populate(existing, target):
+        def mod_add_geo_populate(existing, target, lenient):
             for record in self.expected.records:
                 if record._type != 'A' or record.geo:
-                    existing.records.add(record)
+                    existing.add_record(record)
             record = Record.new(existing, 'simple', {
                 'ttl': 61,
                 'type': 'A',
@@ -514,22 +527,12 @@ class TestRoute53Provider(TestCase):
                     'OC': ['3.2.3.4', '4.2.3.4'],
                 }
             })
-            existing.records.add(record)
+            existing.add_record(record)
 
         provider.populate = mod_add_geo_populate
         change_resource_record_sets_params = {
             'ChangeBatch': {
                 'Changes': [{
-                    'Action': 'DELETE',
-                    'ResourceRecordSet': {
-                        'GeoLocation': {'ContinentCode': 'OC'},
-                        'Name': 'simple.unit.tests.',
-                        'ResourceRecords': [{'Value': '3.2.3.4'},
-                                            {'Value': '4.2.3.4'}],
-                        'SetIdentifier': 'OC',
-                        'TTL': 61,
-                        'Type': 'A'}
-                }, {
                     'Action': 'DELETE',
                     'ResourceRecordSet': {
                         'GeoLocation': {'CountryCode': '*'},
@@ -539,6 +542,16 @@ class TestRoute53Provider(TestCase):
                         'SetIdentifier': 'default',
                         'TTL': 61,
                         'Type': 'A'}
+                }, {
+                    'Action': 'DELETE',
+                    'ResourceRecordSet': {
+                        'GeoLocation': {'ContinentCode': 'OC'},
+                        'Name': 'simple.unit.tests.',
+                        'ResourceRecords': [{'Value': '3.2.3.4'},
+                                            {'Value': '4.2.3.4'}],
+                        'SetIdentifier': 'OC',
+                        'TTL': 61,
+                        'Type': 'A'}
                 }, {
                     'Action': 'CREATE',
                     'ResourceRecordSet': {
@@ -579,7 +592,7 @@ class TestRoute53Provider(TestCase):
                              {})
 
         plan = provider.plan(self.expected)
-        self.assertEquals(8, len(plan.changes))
+        self.assertEquals(9, len(plan.changes))
         for change in plan.changes:
             self.assertIsInstance(change, Create)
         stubber.assert_no_pending_responses()
@@ -626,7 +639,7 @@ class TestRoute53Provider(TestCase):
                                  'SubmittedAt': '2017-01-29T01:02:03Z',
                              }}, {'HostedZoneId': 'z42', 'ChangeBatch': ANY})
 
-        self.assertEquals(8, provider.apply(plan))
+        self.assertEquals(9, provider.apply(plan))
         stubber.assert_no_pending_responses()
 
     def test_health_checks_pagination(self):
@@ -694,8 +707,7 @@ class TestRoute53Provider(TestCase):
                 'AF': ['4.2.3.4'],
             }
         })
-        id = provider._get_health_check_id(record, 'AF', record.geo['AF'],
-                                           True)
+        id = provider.get_health_check_id(record, 'AF', record.geo['AF'], True)
         self.assertEquals('42', id)
 
     def test_health_check_create(self):
@@ -765,13 +777,12 @@ class TestRoute53Provider(TestCase):
         })
 
         # if not allowed to create returns none
-        id = provider._get_health_check_id(record, 'AF', record.geo['AF'],
-                                           False)
+        id = provider.get_health_check_id(record, 'AF', record.geo['AF'],
+                                          False)
         self.assertFalse(id)
 
         # when allowed to create we do
-        id = provider._get_health_check_id(record, 'AF', record.geo['AF'],
-                                           True)
+        id = provider.get_health_check_id(record, 'AF', record.geo['AF'], True)
         self.assertEquals('42', id)
         stubber.assert_no_pending_responses()
 
@@ -1106,10 +1117,6 @@ class TestRoute53Provider(TestCase):
         self.assertEquals(0, len(extra))
         stubber.assert_no_pending_responses()
 
-    def test_route_53_record(self):
-        # Just make sure it doesn't blow up
-        _Route53Record('foo.unit.tests.', 'A', 30).__repr__()
-
     def _get_test_plan(self, max_changes):
 
         provider = Route53Provider('test', 'abc', '123', max_changes)
@@ -1180,16 +1187,16 @@ class TestRoute53Provider(TestCase):
     @patch('octodns.provider.route53.Route53Provider._really_apply')
     def test_apply_1(self, really_apply_mock):
 
-        # 17 RRs with max of 18 should only get applied in one call
-        provider, plan = self._get_test_plan(18)
+        # 18 RRs with max of 19 should only get applied in one call
+        provider, plan = self._get_test_plan(19)
         provider.apply(plan)
         really_apply_mock.assert_called_once()
 
     @patch('octodns.provider.route53.Route53Provider._really_apply')
     def test_apply_2(self, really_apply_mock):
 
-        # 17 RRs with max of 17 should only get applied in two calls
-        provider, plan = self._get_test_plan(17)
+        # 18 RRs with max of 17 should only get applied in two calls
+        provider, plan = self._get_test_plan(18)
         provider.apply(plan)
         self.assertEquals(2, really_apply_mock.call_count)
 
@@ -1237,3 +1244,81 @@ class TestRoute53Provider(TestCase):
             'TTL': 30,
             'Type': 'TXT',
         }))
+
+    def test_client_max_attempts(self):
+        provider = Route53Provider('test', 'abc', '123',
+                                   client_max_attempts=42)
+        # NOTE: this will break if boto ever changes the impl details...
+        self.assertEquals(43, provider._conn.meta.events
+                          ._unique_id_handlers['retry-config-route53']
+                          ['handler']._checker.__dict__['_max_attempts'])
+
+
+class TestRoute53Records(TestCase):
+
+    def test_route53_record(self):
+        existing = Zone('unit.tests.', [])
+        record_a = Record.new(existing, '', {
+            'geo': {
+                'NA-US': ['2.2.2.2', '3.3.3.3'],
+                'OC': ['4.4.4.4', '5.5.5.5']
+            },
+            'ttl': 99,
+            'type': 'A',
+            'values': ['9.9.9.9']
+        })
+        a = _Route53Record(None, record_a, False)
+        self.assertEquals(a, a)
+        b = _Route53Record(None, Record.new(existing, '',
+                                            {'ttl': 32, 'type': 'A',
+                                             'values': ['8.8.8.8',
+                                                        '1.1.1.1']}),
+                           False)
+        self.assertEquals(b, b)
+        c = _Route53Record(None, Record.new(existing, 'other',
+                                            {'ttl': 99, 'type': 'A',
+                                             'values': ['9.9.9.9']}),
+                           False)
+        self.assertEquals(c, c)
+        d = _Route53Record(None, Record.new(existing, '',
+                                            {'ttl': 42, 'type': 'MX',
+                                             'value': {
+                                                 'preference': 10,
+                                                 'exchange': 'foo.bar.'}}),
+                           False)
+        self.assertEquals(d, d)
+
+        # Same fqdn & type is same record
+        self.assertEquals(a, b)
+        # Same name & different type is not the same
+        self.assertNotEquals(a, d)
+        # Different name & same type is not the same
+        self.assertNotEquals(a, c)
+
+        # Same everything, different class is not the same
+        e = _Route53GeoDefault(None, record_a, False)
+        self.assertNotEquals(a, e)
+
+        class DummyProvider(object):
+
+            def get_health_check_id(self, *args, **kwargs):
+                return None
+
+        provider = DummyProvider()
+        f = _Route53GeoRecord(provider, record_a, 'NA-US',
+                              record_a.geo['NA-US'], False)
+        self.assertEquals(f, f)
+        g = _Route53GeoRecord(provider, record_a, 'OC',
+                              record_a.geo['OC'], False)
+        self.assertEquals(g, g)
+
+        # Geo and non-geo are not the same, using Geo as primary to get it's
+        # __cmp__
+        self.assertNotEquals(f, a)
+        # Same everything, different geo's is not the same
+        self.assertNotEquals(f, g)
+
+        # Make sure it doesn't blow up
+        a.__repr__()
+        e.__repr__()
+        f.__repr__()

tests/test_octodns_provider_yaml.py

@@ -30,7 +30,7 @@ class TestYamlProvider(TestCase):
 
         # without it we see everything
         source.populate(zone)
-        self.assertEquals(15, len(zone.records))
+        self.assertEquals(16, len(zone.records))
 
         # Assumption here is that a clean round-trip means that everything
         # worked as expected, data that went in came back out and could be
@@ -100,6 +100,12 @@ class TestYamlProvider(TestCase):
         with self.assertRaises(ConstructorError):
             source.populate(zone)
 
+        source = YamlProvider('test', join(dirname(__file__), 'config'),
+                              enforce_order=False)
+        # no exception
+        source.populate(zone)
+        self.assertEqual(2, len(zone.records))
+
     def test_subzone_handling(self):
         source = YamlProvider('test', join(dirname(__file__), 'config'))
 

tests/test_octodns_source_tinydns.py

@@ -68,22 +68,22 @@ class TestTinyDnsFileSource(TestCase):
                 'type': 'MX',
                 'ttl': 3600,
                 'values': [{
-                    'priority': 10,
-                    'value': 'smtp-1-host.example.com.',
+                    'preference': 10,
+                    'exchange': 'smtp-1-host.example.com.',
                 }, {
-                    'priority': 20,
-                    'value': 'smtp-2-host.example.com.',
+                    'preference': 20,
+                    'exchange': 'smtp-2-host.example.com.',
                 }]
             }),
             ('smtp', {
                 'type': 'MX',
                 'ttl': 1800,
                 'values': [{
-                    'priority': 30,
-                    'value': 'smtp-1-host.example.com.',
+                    'preference': 30,
+                    'exchange': 'smtp-1-host.example.com.',
                 }, {
-                    'priority': 40,
-                    'value': 'smtp-2-host.example.com.',
+                    'preference': 40,
+                    'exchange': 'smtp-2-host.example.com.',
                 }]
             }),
         ):

tests/test_octodns_yaml.py

@@ -48,8 +48,8 @@ class TestYaml(TestCase):
 '*.11.2': 'd'
 '*.10.1': 'c'
 ''')
-        self.assertEquals('keys out of order: *.2.2, *.1.2, *.11.2, *.10.1',
-                          ctx.exception.problem)
+        self.assertTrue('keys out of order: expected *.1.2 got *.2.2 at' in
+                        ctx.exception.problem)
 
         buf = StringIO()
         safe_dump({
@@ -59,3 +59,12 @@ class TestYaml(TestCase):
         }, buf)
         self.assertEquals("---\n'*.1.1': 42\n'*.2.1': 44\n'*.11.1': 43\n",
                           buf.getvalue())
+
+        # hex sorting isn't ideal, not treated as hex, this make sure we don't
+        # change the behavior
+        buf = StringIO()
+        safe_dump({
+            '45a03129': 42,
+            '45a0392a': 43,
+        }, buf)
+        self.assertEquals("---\n45a0392a: 43\n45a03129: 42\n", buf.getvalue())

tests/test_octodns_zone.py

@@ -8,7 +8,8 @@ from __future__ import absolute_import, division, print_function, \
 from unittest import TestCase
 
 from octodns.record import ARecord, AaaaRecord, Create, Delete, Record, Update
-from octodns.zone import DuplicateRecordException, SubzoneRecordException, Zone
+from octodns.zone import DuplicateRecordException, InvalidNodeException, \
+    SubzoneRecordException, Zone
 
 from helpers import SimpleProvider
 
@@ -38,6 +39,7 @@ class TestZone(TestCase):
 
         a = ARecord(zone, 'a', {'ttl': 42, 'value': '1.1.1.1'})
         b = ARecord(zone, 'b', {'ttl': 42, 'value': '1.1.1.1'})
+        c = ARecord(zone, 'a', {'ttl': 43, 'value': '2.2.2.2'})
 
         zone.add_record(a)
         self.assertEquals(zone.records, set([a]))
@@ -47,6 +49,11 @@ class TestZone(TestCase):
         self.assertEquals('Duplicate record a.unit.tests., type A',
                           ctx.exception.message)
         self.assertEquals(zone.records, set([a]))
+
+        # can add duplicate with replace=True
+        zone.add_record(c, replace=True)
+        self.assertEquals('2.2.2.2', list(zone.records)[0].values[0])
+
         # Can add dup name, with different type
         zone.add_record(b)
         self.assertEquals(zone.records, set([a, b]))
@@ -70,7 +77,7 @@ class TestZone(TestCase):
         # add a record, delete a record -> [Delete, Create]
         c = ARecord(before, 'c', {'ttl': 42, 'value': '1.1.1.1'})
         after.add_record(c)
-        after.records.remove(b)
+        after._remove_record(b)
         self.assertEquals(after.records, set([a, c]))
         changes = before.changes(after, target)
         self.assertEquals(2, len(changes))
@@ -205,3 +212,27 @@ class TestZone(TestCase):
 
         self.assertTrue(zone_missing.changes(zone_normal, provider))
         self.assertFalse(zone_missing.changes(zone_ignored, provider))
+
+    def test_cname_coexisting(self):
+        zone = Zone('unit.tests.', [])
+        a = Record.new(zone, 'www', {
+            'ttl': 60,
+            'type': 'A',
+            'value': '9.9.9.9',
+        })
+        cname = Record.new(zone, 'www', {
+            'ttl': 60,
+            'type': 'CNAME',
+            'value': 'foo.bar.com.',
+        })
+
+        # add cname to a
+        zone.add_record(a)
+        with self.assertRaises(InvalidNodeException):
+            zone.add_record(cname)
+
+        # add a to cname
+        zone = Zone('unit.tests.', [])
+        zone.add_record(cname)
+        with self.assertRaises(InvalidNodeException):
+            zone.add_record(a)