Merge branch 'master' into master

.gitignore

@@ -1,6 +1,7 @@
 *.pyc
 .coverage
 .env
+/config/
 coverage.xml
 dist/
 env/
@@ -9,5 +10,3 @@ nosetests.xml
 octodns.egg-info/
 output/
 tmp/
-build/
-config/

README.md

@@ -150,7 +150,7 @@ The above command pulled the existing data out of Route53 and placed the results
 | Provider | Record Support | GeoDNS Support | Notes |
 |--|--|--|--|
 | [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | |
-| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted |
+| [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted |
 | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted |
 | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted |
 | [DynProvider](/octodns/provider/dyn.py) | All | Yes | |

octodns/manager.py

@@ -5,13 +5,13 @@
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
-from StringIO import StringIO
 from concurrent.futures import ThreadPoolExecutor
 from importlib import import_module
 from os import environ
 import logging
 
-from .provider.base import BaseProvider, Plan
+from .provider.base import BaseProvider
+from .provider.plan import Plan
 from .provider.yaml import YamlProvider
 from .record import Record
 from .yaml import safe_load
@@ -95,23 +95,8 @@ class Manager(object):
                 self.log.exception('Invalid provider class')
                 raise Exception('Provider {} is missing class'
                                 .format(provider_name))
-            _class = self._get_provider_class(_class)
-            # Build up the arguments we need to pass to the provider
-            kwargs = {}
-            for k, v in provider_config.items():
-                try:
-                    if v.startswith('env/'):
-                        try:
-                            env_var = v[4:]
-                            v = environ[env_var]
-                        except KeyError:
-                            self.log.exception('Invalid provider config')
-                            raise Exception('Incorrect provider config, '
-                                            'missing env var {}'
-                                            .format(env_var))
-                except AttributeError:
-                    pass
-                kwargs[k] = v
+            _class = self._get_named_class('provider', _class)
+            kwargs = self._build_kwargs(provider_config)
             try:
                 self.providers[provider_name] = _class(provider_name, **kwargs)
             except TypeError:
@@ -139,20 +124,64 @@ class Manager(object):
                     where = where[piece]
         self.zone_tree = zone_tree
 
-    def _get_provider_class(self, _class):
+        self.plan_outputs = {}
+        plan_outputs = manager_config.get('plan_outputs', {
+            'logger': {
+                'class': 'octodns.provider.plan.PlanLogger',
+                'level': 'info'
+            }
+        })
+        for plan_output_name, plan_output_config in plan_outputs.items():
+            try:
+                _class = plan_output_config.pop('class')
+            except KeyError:
+                self.log.exception('Invalid plan_output class')
+                raise Exception('plan_output {} is missing class'
+                                .format(plan_output_name))
+            _class = self._get_named_class('plan_output', _class)
+            kwargs = self._build_kwargs(plan_output_config)
+            try:
+                self.plan_outputs[plan_output_name] = \
+                    _class(plan_output_name, **kwargs)
+            except TypeError:
+                self.log.exception('Invalid plan_output config')
+                raise Exception('Incorrect plan_output config for {}'
+                                .format(plan_output_name))
+
+    def _get_named_class(self, _type, _class):
         try:
             module_name, class_name = _class.rsplit('.', 1)
             module = import_module(module_name)
         except (ImportError, ValueError):
-            self.log.exception('_get_provider_class: Unable to import '
+            self.log.exception('_get_{}_class: Unable to import '
                                'module %s', _class)
-            raise Exception('Unknown provider class: {}'.format(_class))
+            raise Exception('Unknown {} class: {}'.format(_type, _class))
         try:
             return getattr(module, class_name)
         except AttributeError:
-            self.log.exception('_get_provider_class: Unable to get class %s '
+            self.log.exception('_get_{}_class: Unable to get class %s '
                                'from module %s', class_name, module)
-            raise Exception('Unknown provider class: {}'.format(_class))
+            raise Exception('Unknown {} class: {}'.format(_type, _class))
+
+    def _build_kwargs(self, source):
+        # Build up the arguments we need to pass to the provider
+        kwargs = {}
+        for k, v in source.items():
+            try:
+                if v.startswith('env/'):
+                    try:
+                        env_var = v[4:]
+                        v = environ[env_var]
+                    except KeyError:
+                        self.log.exception('Invalid provider config')
+                        raise Exception('Incorrect provider config, '
+                                        'missing env var {}'
+                                        .format(env_var))
+            except AttributeError:
+                pass
+            kwargs[k] = v
+
+        return kwargs
 
     def configured_sub_zones(self, zone_name):
         # Reversed pieces of the zone name
@@ -259,39 +288,8 @@ class Manager(object):
         # plan pairs.
         plans = [p for f in futures for p in f.result()]
 
-        hr = '*************************************************************' \
-            '*******************\n'
-        buf = StringIO()
-        buf.write('\n')
-        if plans:
-            current_zone = None
-            for target, plan in plans:
-                if plan.desired.name != current_zone:
-                    current_zone = plan.desired.name
-                    buf.write(hr)
-                    buf.write('* ')
-                    buf.write(current_zone)
-                    buf.write('\n')
-                    buf.write(hr)
-
-                buf.write('* ')
-                buf.write(target.id)
-                buf.write(' (')
-                buf.write(target)
-                buf.write(')\n*   ')
-                for change in plan.changes:
-                    buf.write(change.__repr__(leader='* '))
-                    buf.write('\n*   ')
-
-                buf.write('Summary: ')
-                buf.write(plan)
-                buf.write('\n')
-        else:
-            buf.write(hr)
-            buf.write('No changes were planned\n')
-        buf.write(hr)
-        buf.write('\n')
-        self.log.info(buf.getvalue())
+        for output in self.plan_outputs.values():
+            output.run(plans=plans, log=self.log)
 
         if not force:
             self.log.debug('sync:   checking safety')

octodns/provider/base.py

@@ -7,78 +7,7 @@ from __future__ import absolute_import, division, print_function, \
 
 from ..source.base import BaseSource
 from ..zone import Zone
-from logging import getLogger
-
-
-class UnsafePlan(Exception):
-    pass
-
-
-class Plan(object):
-    log = getLogger('Plan')
-
-    MAX_SAFE_UPDATE_PCENT = .3
-    MAX_SAFE_DELETE_PCENT = .3
-    MIN_EXISTING_RECORDS = 10
-
-    def __init__(self, existing, desired, changes,
-                 update_pcent_threshold=MAX_SAFE_UPDATE_PCENT,
-                 delete_pcent_threshold=MAX_SAFE_DELETE_PCENT):
-        self.existing = existing
-        self.desired = desired
-        self.changes = changes
-        self.update_pcent_threshold = update_pcent_threshold
-        self.delete_pcent_threshold = delete_pcent_threshold
-
-        change_counts = {
-            'Create': 0,
-            'Delete': 0,
-            'Update': 0
-        }
-        for change in changes:
-            change_counts[change.__class__.__name__] += 1
-        self.change_counts = change_counts
-
-        try:
-            existing_n = len(self.existing.records)
-        except AttributeError:
-            existing_n = 0
-
-        self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d'
-                       'Existing=%d',
-                       self.change_counts['Create'],
-                       self.change_counts['Update'],
-                       self.change_counts['Delete'], existing_n)
-
-    def raise_if_unsafe(self):
-        # TODO: what is safe really?
-        if self.existing and \
-           len(self.existing.records) >= self.MIN_EXISTING_RECORDS:
-
-            existing_record_count = len(self.existing.records)
-            update_pcent = self.change_counts['Update'] / existing_record_count
-            delete_pcent = self.change_counts['Delete'] / existing_record_count
-
-            if update_pcent > self.update_pcent_threshold:
-                raise UnsafePlan('Too many updates, {} is over {} percent'
-                                 '({}/{})'.format(
-                                     update_pcent,
-                                     self.MAX_SAFE_UPDATE_PCENT * 100,
-                                     self.change_counts['Update'],
-                                     existing_record_count))
-            if delete_pcent > self.delete_pcent_threshold:
-                raise UnsafePlan('Too many deletes, {} is over {} percent'
-                                 '({}/{})'.format(
-                                     delete_pcent,
-                                     self.MAX_SAFE_DELETE_PCENT * 100,
-                                     self.change_counts['Delete'],
-                                     existing_record_count))
-
-    def __repr__(self):
-        return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \
-            .format(self.change_counts['Create'], self.change_counts['Update'],
-                    self.change_counts['Delete'],
-                    len(self.existing.records))
+from .plan import Plan
 
 
 class BaseProvider(BaseSource):

octodns/provider/cloudflare.py

@@ -7,6 +7,7 @@ from __future__ import absolute_import, division, print_function, \
 
 from collections import defaultdict
 from logging import getLogger
+from json import dumps
 from requests import Session
 
 from ..record import Record, Update
@@ -36,7 +37,8 @@ class CloudflareProvider(BaseProvider):
     '''
     SUPPORTS_GEO = False
     # TODO: support SRV
-    SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SPF', 'TXT'))
+    SUPPORTS = set(('ALIAS', 'A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'SPF',
+                    'TXT'))
 
     MIN_TTL = 120
     TIMEOUT = 15
@@ -123,6 +125,8 @@ class CloudflareProvider(BaseProvider):
             'value': '{}.'.format(only['content'])
         }
 
+    _data_for_ALIAS = _data_for_CNAME
+
     def _data_for_MX(self, _type, records):
         values = []
         for r in records:
@@ -181,6 +185,11 @@ class CloudflareProvider(BaseProvider):
 
             for name, types in values.items():
                 for _type, records in types.items():
+
+                    # Cloudflare supports ALIAS semantics with root CNAMEs
+                    if _type == 'CNAME' and name == '':
+                        _type = 'ALIAS'
+
                     data_for = getattr(self, '_data_for_{}'.format(_type))
                     data = data_for(_type, records)
                     record = Record.new(zone, name, data, source=self,
@@ -232,25 +241,111 @@ class CloudflareProvider(BaseProvider):
                 'content': value.exchange
             }
 
+    def _gen_contents(self, record):
+        name = record.fqdn[:-1]
+        _type = record._type
+        ttl = max(self.MIN_TTL, record.ttl)
+
+        # Cloudflare supports ALIAS semantics with a root CNAME
+        if _type == 'ALIAS':
+            _type = 'CNAME'
+
+        contents_for = getattr(self, '_contents_for_{}'.format(_type))
+        for content in contents_for(record):
+            content.update({
+                'name': name,
+                'type': _type,
+                'ttl': ttl,
+            })
+            yield content
+
     def _apply_Create(self, change):
         new = change.new
         zone_id = self.zones[new.zone.name]
-        contents_for = getattr(self, '_contents_for_{}'.format(new._type))
         path = '/zones/{}/dns_records'.format(zone_id)
-        name = new.fqdn[:-1]
-        for content in contents_for(change.new):
-            content.update({
-                'name': name,
-                'type': new._type,
-                # Cloudflare has a min ttl of 120s
-                'ttl': max(self.MIN_TTL, new.ttl),
-            })
+        for content in self._gen_contents(new):
             self._request('POST', path, data=content)
 
+    def _hash_content(self, content):
+        # Some of the dicts are nested so this seems about as good as any
+        # option we have for consistently hashing them (within a single run)
+        return hash(dumps(content, sort_keys=True))
+
     def _apply_Update(self, change):
-        # Create the new and delete the old
-        self._apply_Create(change)
-        self._apply_Delete(change)
+
+        # Ugh, this is pretty complicated and ugly, mainly due to the
+        # sub-optimal API/semantics. Ideally we'd have a batch change API like
+        # Route53's to make this 100% clean and safe without all this PITA, but
+        # we don't so we'll have to work around that and manually do it as
+        # safely as possible. Note this still isn't perfect as we don't/can't
+        # practically take into account things like the different "types" of
+        # CAA records so when we "swap" there may be brief periods where things
+        # are invalid or even worse Cloudflare may update their validations to
+        # prevent dups. I see no clean way around that short of making this
+        # understand 100% of the details of each record type and develop an
+        # individual/specific ordering of changes that prevents it. That'd
+        # probably result in more code than this whole provider currently has
+        # so... :-(
+
+        existing_contents = {
+            self._hash_content(c): c
+            for c in self._gen_contents(change.existing)
+        }
+        new_contents = {
+            self._hash_content(c): c
+            for c in self._gen_contents(change.new)
+        }
+
+        # We need a list of keys to consider for diffs, use the first content
+        # before we muck with anything
+        keys = existing_contents.values()[0].keys()
+
+        # Find the things we need to add
+        adds = []
+        for k, content in new_contents.items():
+            try:
+                existing_contents.pop(k)
+                self.log.debug('_apply_Update: leaving %s', content)
+            except KeyError:
+                adds.append(content)
+
+        zone_id = self.zones[change.new.zone.name]
+
+        # Find things we need to remove
+        name = change.new.fqdn[:-1]
+        _type = change.new._type
+        # OK, work through each record from the zone
+        for record in self.zone_records(change.new.zone):
+            if name == record['name'] and _type == record['type']:
+                # This is match for our name and type, we need to look at
+                # contents now, build a dict of the relevant keys and vals
+                content = {}
+                for k in keys:
+                    content[k] = record[k]
+                # :-(
+                if _type in ('CNAME', 'MX', 'NS'):
+                    content['content'] += '.'
+                # If the hash of that dict isn't in new this record isn't
+                # needed
+                if self._hash_content(content) not in new_contents:
+                    rid = record['id']
+                    path = '/zones/{}/dns_records/{}'.format(record['zone_id'],
+                                                             rid)
+                    try:
+                        add_content = adds.pop(0)
+                        self.log.debug('_apply_Update: swapping %s -> %s, %s',
+                                       content, add_content, rid)
+                        self._request('PUT', path, data=add_content)
+                    except IndexError:
+                        self.log.debug('_apply_Update: removing %s, %s',
+                                       content, rid)
+                        self._request('DELETE', path)
+
+        # Any remaining adds just need to be created
+        path = '/zones/{}/dns_records'.format(zone_id)
+        for content in adds:
+            self.log.debug('_apply_Update: adding %s', content)
+            self._request('POST', path, data=content)
 
     def _apply_Delete(self, change):
         existing = change.existing

octodns/provider/plan.py

@@ -0,0 +1,266 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from StringIO import StringIO
+from logging import DEBUG, ERROR, INFO, WARN, getLogger
+from sys import stdout
+
+
+class UnsafePlan(Exception):
+    pass
+
+
+class Plan(object):
+    log = getLogger('Plan')
+
+    MAX_SAFE_UPDATE_PCENT = .3
+    MAX_SAFE_DELETE_PCENT = .3
+    MIN_EXISTING_RECORDS = 10
+
+    def __init__(self, existing, desired, changes,
+                 update_pcent_threshold=MAX_SAFE_UPDATE_PCENT,
+                 delete_pcent_threshold=MAX_SAFE_DELETE_PCENT):
+        self.existing = existing
+        self.desired = desired
+        self.changes = changes
+        self.update_pcent_threshold = update_pcent_threshold
+        self.delete_pcent_threshold = delete_pcent_threshold
+
+        change_counts = {
+            'Create': 0,
+            'Delete': 0,
+            'Update': 0
+        }
+        for change in changes:
+            change_counts[change.__class__.__name__] += 1
+        self.change_counts = change_counts
+
+        try:
+            existing_n = len(self.existing.records)
+        except AttributeError:
+            existing_n = 0
+
+        self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d'
+                       'Existing=%d',
+                       self.change_counts['Create'],
+                       self.change_counts['Update'],
+                       self.change_counts['Delete'], existing_n)
+
+    def raise_if_unsafe(self):
+        # TODO: what is safe really?
+        if self.existing and \
+           len(self.existing.records) >= self.MIN_EXISTING_RECORDS:
+
+            existing_record_count = len(self.existing.records)
+            update_pcent = self.change_counts['Update'] / existing_record_count
+            delete_pcent = self.change_counts['Delete'] / existing_record_count
+
+            if update_pcent > self.update_pcent_threshold:
+                raise UnsafePlan('Too many updates, {} is over {} percent'
+                                 '({}/{})'.format(
+                                     update_pcent,
+                                     self.MAX_SAFE_UPDATE_PCENT * 100,
+                                     self.change_counts['Update'],
+                                     existing_record_count))
+            if delete_pcent > self.delete_pcent_threshold:
+                raise UnsafePlan('Too many deletes, {} is over {} percent'
+                                 '({}/{})'.format(
+                                     delete_pcent,
+                                     self.MAX_SAFE_DELETE_PCENT * 100,
+                                     self.change_counts['Delete'],
+                                     existing_record_count))
+
+    def __repr__(self):
+        return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \
+            .format(self.change_counts['Create'], self.change_counts['Update'],
+                    self.change_counts['Delete'],
+                    len(self.existing.records))
+
+
+class _PlanOutput(object):
+
+    def __init__(self, name):
+        self.name = name
+
+
+class PlanLogger(_PlanOutput):
+
+    def __init__(self, name, level='info'):
+        super(PlanLogger, self).__init__(name)
+        try:
+            self.level = {
+                'debug': DEBUG,
+                'info': INFO,
+                'warn': WARN,
+                'warning': WARN,
+                'error': ERROR
+            }[level.lower()]
+        except (AttributeError, KeyError):
+            raise Exception('Unsupported level: {}'.format(level))
+
+    def run(self, log, plans, *args, **kwargs):
+        hr = '*************************************************************' \
+            '*******************\n'
+        buf = StringIO()
+        buf.write('\n')
+        if plans:
+            current_zone = None
+            for target, plan in plans:
+                if plan.desired.name != current_zone:
+                    current_zone = plan.desired.name
+                    buf.write(hr)
+                    buf.write('* ')
+                    buf.write(current_zone)
+                    buf.write('\n')
+                    buf.write(hr)
+
+                buf.write('* ')
+                buf.write(target.id)
+                buf.write(' (')
+                buf.write(target)
+                buf.write(')\n*   ')
+                for change in plan.changes:
+                    buf.write(change.__repr__(leader='* '))
+                    buf.write('\n*   ')
+
+                buf.write('Summary: ')
+                buf.write(plan)
+                buf.write('\n')
+        else:
+            buf.write(hr)
+            buf.write('No changes were planned\n')
+        buf.write(hr)
+        buf.write('\n')
+        log.log(self.level, buf.getvalue())
+
+
+def _value_stringifier(record, sep):
+    try:
+        values = [str(v) for v in record.values]
+    except AttributeError:
+        values = [record.value]
+    for code, gv in sorted(getattr(record, 'geo', {}).items()):
+        vs = ', '.join([str(v) for v in gv.values])
+        values.append('{}: {}'.format(code, vs))
+    return sep.join(values)
+
+
+class PlanMarkdown(_PlanOutput):
+
+    def run(self, plans, fh=stdout, *args, **kwargs):
+        if plans:
+            current_zone = None
+            for target, plan in plans:
+                if plan.desired.name != current_zone:
+                    current_zone = plan.desired.name
+                    fh.write('## ')
+                    fh.write(current_zone)
+                    fh.write('\n\n')
+
+                fh.write('### ')
+                fh.write(target.id)
+                fh.write('\n\n')
+
+                fh.write('| Operation | Name | Type | TTL | Value | Source |\n'
+                         '|--|--|--|--|--|--|\n')
+
+                for change in plan.changes:
+                    existing = change.existing
+                    new = change.new
+                    record = change.record
+                    fh.write('| ')
+                    fh.write(change.__class__.__name__)
+                    fh.write(' | ')
+                    fh.write(record.name)
+                    fh.write(' | ')
+                    fh.write(record._type)
+                    fh.write(' | ')
+                    # TTL
+                    if existing:
+                        fh.write(str(existing.ttl))
+                        fh.write(' | ')
+                        fh.write(_value_stringifier(existing, '; '))
+                        fh.write(' | |\n')
+                        if new:
+                            fh.write('| | | | ')
+
+                    if new:
+                        fh.write(str(new.ttl))
+                        fh.write(' | ')
+                        fh.write(_value_stringifier(new, '; '))
+                        fh.write(' | ')
+                        fh.write(new.source.id)
+                        fh.write(' |\n')
+
+                fh.write('\nSummary: ')
+                fh.write(str(plan))
+                fh.write('\n\n')
+        else:
+            fh.write('## No changes were planned\n')
+
+
+class PlanHtml(_PlanOutput):
+
+    def run(self, plans, fh=stdout, *args, **kwargs):
+        if plans:
+            current_zone = None
+            for target, plan in plans:
+                if plan.desired.name != current_zone:
+                    current_zone = plan.desired.name
+                    fh.write('<h2>')
+                    fh.write(current_zone)
+                    fh.write('</h2>\n')
+
+                fh.write('<h3>')
+                fh.write(target.id)
+                fh.write('''</h3>
+<table>
+  <tr>
+    <th>Operation</th>
+    <th>Name</th>
+    <th>Type</th>
+    <th>TTL</th>
+    <th>Value</th>
+    <th>Source</th>
+  </tr>
+''')
+
+                for change in plan.changes:
+                    existing = change.existing
+                    new = change.new
+                    record = change.record
+                    fh.write('  <tr>\n    <td>')
+                    fh.write(change.__class__.__name__)
+                    fh.write('</td>\n    <td>')
+                    fh.write(record.name)
+                    fh.write('</td>\n    <td>')
+                    fh.write(record._type)
+                    fh.write('</td>\n')
+                    # TTL
+                    if existing:
+                        fh.write('    <td>')
+                        fh.write(str(existing.ttl))
+                        fh.write('</td>\n    <td>')
+                        fh.write(_value_stringifier(existing, '<br/>'))
+                        fh.write('</td>\n    <td></td>\n  </tr>\n')
+                        if new:
+                            fh.write('  <tr>\n    <td colspan=3></td>\n')
+
+                    if new:
+                        fh.write('    <td>')
+                        fh.write(str(new.ttl))
+                        fh.write('</td>\n    <td>')
+                        fh.write(_value_stringifier(new, '<br/>'))
+                        fh.write('</td>\n    <td>')
+                        fh.write(new.source.id)
+                        fh.write('</td>\n  </tr>\n')
+
+                fh.write('  <tr>\n    <td colspan=6>Summary: ')
+                fh.write(str(plan))
+                fh.write('</td>\n  </tr>\n</table>\n')
+        else:
+            fh.write('<b>No changes were planned</b>')

octodns/provider/route53.py

@@ -385,10 +385,10 @@ class Route53Provider(BaseProvider):
             values.append({
                 'order': order,
                 'preference': preference,
-                'flags': flags if flags else None,
-                'service': service if service else None,
-                'regexp': regexp if regexp else None,
-                'replacement': replacement if replacement else None,
+                'flags': flags,
+                'service': service,
+                'regexp': regexp,
+                'replacement': replacement,
             })
         return {
             'type': rrset['Type'],

setup.cfg

@@ -22,12 +22,12 @@ classifiers =
 install_requires = 
     PyYaml>=3.12
     dnspython>=1.15.0
-    futures==3.1.1
-    incf.countryutils==1.0
-    ipaddress==1.0.18
-    natsort==5.0.3
-    python-dateutil==2.6.1
-    requests==2.13.0
+    futures>=3.1.1
+    incf.countryutils>=1.0
+    ipaddress>=1.0.18
+    natsort>=5.0.3
+    python-dateutil>=2.6.1
+    requests>=2.13.0
 packages = find:
 include_package_data = True
 
@@ -47,17 +47,17 @@ exclude =
 dev = 
     azure-mgmt-dns==1.0.1
     azure-common==1.1.6
-    boto3==1.4.6
-    botocore==1.6.8
-    docutils==0.14
-    dyn==1.8.0
-    google-cloud==0.27.0
-    jmespath==0.9.3
+    boto3>=1.4.6
+    botocore>=1.6.8
+    docutils>=0.14
+    dyn>=1.8.0
+    google-cloud>=0.27.0
+    jmespath>=0.9.3
     msrestazure==0.4.10
-    nsone==0.9.14
-    ovh==0.4.7
-    s3transfer==0.1.10
-    six==1.10.0
+    nsone>=0.9.14
+    ovh>=0.4.7
+    s3transfer>=0.1.10
+    six>=1.10.0
 test =
     coverage
     mock

tests/config/bad-plan-output-config.yaml

@@ -0,0 +1,7 @@
+manager:
+  plan_outputs:
+    'bad': 
+      class: octodns.provider.plan.PlanLogger
+      invalid: config
+providers: {}
+zones: {}

tests/config/bad-plan-output-missing-class.yaml

@@ -0,0 +1,5 @@
+manager:
+  plan_outputs:
+    'bad': {}
+providers: {}
+zones: {}

tests/fixtures/cloudflare-dns_records-page-1.json

@@ -180,7 +180,7 @@
     "per_page": 10,
     "total_pages": 2,
     "count": 10,
-    "total_count": 17
+    "total_count": 19
   },
   "success": true,
   "errors": [],

tests/fixtures/cloudflare-dns_records-page-2.json

@@ -163,7 +163,7 @@
     "per_page": 10,
     "total_pages": 2,
     "count": 9,
-    "total_count": 20
+    "total_count": 19
   },
   "success": true,
   "errors": [],

tests/test_octodns_manager.py

@@ -83,6 +83,19 @@ class TestManager(TestCase):
                 .sync(['unknown.target.'])
         self.assertTrue('unknown target' in ctx.exception.message)
 
+    def test_bad_plan_output_class(self):
+        with self.assertRaises(Exception) as ctx:
+            name = 'bad-plan-output-missing-class.yaml'
+            Manager(get_config_filename(name)).sync()
+        self.assertEquals('plan_output bad is missing class',
+                          ctx.exception.message)
+
+    def test_bad_plan_output_config(self):
+        with self.assertRaises(Exception) as ctx:
+            Manager(get_config_filename('bad-plan-output-config.yaml')).sync()
+        self.assertEqual('Incorrect plan_output config for bad',
+                         ctx.exception.message)
+
     def test_source_only_as_a_target(self):
         with self.assertRaises(Exception) as ctx:
             Manager(get_config_filename('unknown-provider.yaml')) \

tests/test_octodns_plan.py

@@ -0,0 +1,92 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from StringIO import StringIO
+from logging import getLogger
+from unittest import TestCase
+
+from octodns.provider.plan import Plan, PlanHtml, PlanLogger, PlanMarkdown
+from octodns.record import Create, Delete, Record, Update
+from octodns.zone import Zone
+
+from helpers import SimpleProvider
+
+
+class TestPlanLogger(TestCase):
+
+    def test_invalid_level(self):
+        with self.assertRaises(Exception) as ctx:
+            PlanLogger('invalid', 'not-a-level')
+        self.assertEquals('Unsupported level: not-a-level',
+                          ctx.exception.message)
+
+
+simple = SimpleProvider()
+zone = Zone('unit.tests.', [])
+existing = Record.new(zone, 'a', {
+    'ttl': 300,
+    'type': 'A',
+    # This matches the zone data above, one to swap, one to leave
+    'values': ['1.1.1.1', '2.2.2.2'],
+})
+new = Record.new(zone, 'a', {
+    'geo': {
+        'AF': ['5.5.5.5'],
+        'NA-US': ['6.6.6.6']
+    },
+    'ttl': 300,
+    'type': 'A',
+    # This leaves one, swaps ones, and adds one
+    'values': ['2.2.2.2', '3.3.3.3', '4.4.4.4'],
+}, simple)
+create = Create(Record.new(zone, 'b', {
+    'ttl': 60,
+    'type': 'CNAME',
+    'value': 'foo.unit.tests.'
+}, simple))
+update = Update(existing, new)
+delete = Delete(new)
+changes = [create, delete, update]
+plans = [
+    (simple, Plan(zone, zone, changes)),
+    (simple, Plan(zone, zone, changes)),
+]
+
+
+class TestPlanHtml(TestCase):
+    log = getLogger('TestPlanHtml')
+
+    def test_empty(self):
+        out = StringIO()
+        PlanHtml('html').run([], fh=out)
+        self.assertEquals('<b>No changes were planned</b>', out.getvalue())
+
+    def test_simple(self):
+        out = StringIO()
+        PlanHtml('html').run(plans, fh=out)
+        out = out.getvalue()
+        self.assertTrue('    <td colspan=6>Summary: Creates=1, Updates=1, '
+                        'Deletes=1, Existing Records=0</td>' in out)
+
+
+class TestPlanMarkdown(TestCase):
+    log = getLogger('TestPlanMarkdown')
+
+    def test_empty(self):
+        out = StringIO()
+        PlanMarkdown('markdown').run([], fh=out)
+        self.assertEquals('## No changes were planned\n', out.getvalue())
+
+    def test_simple(self):
+        out = StringIO()
+        PlanMarkdown('markdown').run(plans, fh=out)
+        out = out.getvalue()
+        self.assertTrue('## unit.tests.' in out)
+        self.assertTrue('Create | b | CNAME | 60 | foo.unit.tests.' in out)
+        self.assertTrue('Update | a | A | 300 | 1.1.1.1;' in out)
+        self.assertTrue('NA-US: 6.6.6.6 | test' in out)
+        self.assertTrue('Delete | a | A | 300 | 2.2.2.2;' in out)

tests/test_octodns_provider_base.py

@@ -9,7 +9,8 @@ from logging import getLogger
 from unittest import TestCase
 
 from octodns.record import Create, Delete, Record, Update
-from octodns.provider.base import BaseProvider, Plan, UnsafePlan
+from octodns.provider.base import BaseProvider
+from octodns.provider.plan import Plan, UnsafePlan
 from octodns.zone import Zone
 
 

tests/test_octodns_provider_cloudflare.py

@@ -11,7 +11,8 @@ from requests import HTTPError
 from requests_mock import ANY, mock as requests_mock
 from unittest import TestCase
 
-from octodns.record import Record
+from octodns.record import Record, Update
+from octodns.provider.base import Plan
 from octodns.provider.cloudflare import CloudflareProvider
 from octodns.provider.yaml import YamlProvider
 from octodns.zone import Zone
@@ -267,15 +268,219 @@ class TestCloudflareProvider(TestCase):
         self.assertEquals(2, provider.apply(plan))
         # recreate for update, and deletes for the 2 parts of the other
         provider._request.assert_has_calls([
-            call('POST', '/zones/42/dns_records', data={
-                'content': '3.2.3.4',
-                'type': 'A',
-                'name': 'ttl.unit.tests',
-                'ttl': 300}),
-            call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/'
-                 'dns_records/fc12ab34cd5611334422ab3322997655'),
+            call('PUT', '/zones/ff12ab34cd5611334422ab3322997650/dns_records/'
+                 'fc12ab34cd5611334422ab3322997655',
+                 data={'content': '3.2.3.4',
+                       'type': 'A',
+                       'name': 'ttl.unit.tests',
+                       'ttl': 300}),
             call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/'
                  'dns_records/fc12ab34cd5611334422ab3322997653'),
             call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/'
                  'dns_records/fc12ab34cd5611334422ab3322997654')
         ])
+
+    def test_update_add_swap(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997653",
+                "type": "A",
+                "name": "a.unit.tests",
+                "content": "1.1.1.1",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            },
+            {
+                "id": "fc12ab34cd5611334422ab3322997654",
+                "type": "A",
+                "name": "a.unit.tests",
+                "content": "2.2.2.2",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            },
+        ])
+
+        provider._request = Mock()
+        provider._request.side_effect = [
+            self.empty,  # no zones
+            {
+                'result': {
+                    'id': 42,
+                }
+            },  # zone create
+            None,
+            None,
+        ]
+
+        # Add something and delete something
+        zone = Zone('unit.tests.', [])
+        existing = Record.new(zone, 'a', {
+            'ttl': 300,
+            'type': 'A',
+            # This matches the zone data above, one to swap, one to leave
+            'values': ['1.1.1.1', '2.2.2.2'],
+        })
+        new = Record.new(zone, 'a', {
+            'ttl': 300,
+            'type': 'A',
+            # This leaves one, swaps ones, and adds one
+            'values': ['2.2.2.2', '3.3.3.3', '4.4.4.4'],
+        })
+        change = Update(existing, new)
+        plan = Plan(zone, zone, [change])
+        provider._apply(plan)
+
+        provider._request.assert_has_calls([
+            call('GET', '/zones', params={'page': 1}),
+            call('POST', '/zones', data={'jump_start': False,
+                                         'name': 'unit.tests'}),
+            call('PUT', '/zones/ff12ab34cd5611334422ab3322997650/dns_records/'
+                 'fc12ab34cd5611334422ab3322997653',
+                 data={'content': '4.4.4.4', 'type': 'A', 'name':
+                       'a.unit.tests', 'ttl': 300}),
+            call('POST', '/zones/42/dns_records',
+                 data={'content': '3.3.3.3', 'type': 'A',
+                       'name': 'a.unit.tests', 'ttl': 300})
+        ])
+
+    def test_update_delete(self):
+        # We need another run so that we can delete, we can't both add and
+        # delete in one go b/c of swaps
+        provider = CloudflareProvider('test', 'email', 'token')
+
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997653",
+                "type": "NS",
+                "name": "unit.tests",
+                "content": "ns1.foo.bar",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            },
+            {
+                "id": "fc12ab34cd5611334422ab3322997654",
+                "type": "NS",
+                "name": "unit.tests",
+                "content": "ns2.foo.bar",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            },
+        ])
+
+        provider._request = Mock()
+        provider._request.side_effect = [
+            self.empty,  # no zones
+            {
+                'result': {
+                    'id': 42,
+                }
+            },  # zone create
+            None,
+            None,
+        ]
+
+        # Add something and delete something
+        zone = Zone('unit.tests.', [])
+        existing = Record.new(zone, '', {
+            'ttl': 300,
+            'type': 'NS',
+            # This matches the zone data above, one to delete, one to leave
+            'values': ['ns1.foo.bar.', 'ns2.foo.bar.'],
+        })
+        new = Record.new(zone, '', {
+            'ttl': 300,
+            'type': 'NS',
+            # This leaves one and deletes one
+            'value': 'ns2.foo.bar.',
+        })
+        change = Update(existing, new)
+        plan = Plan(zone, zone, [change])
+        provider._apply(plan)
+
+        provider._request.assert_has_calls([
+            call('GET', '/zones', params={'page': 1}),
+            call('POST', '/zones',
+                 data={'jump_start': False, 'name': 'unit.tests'}),
+            call('DELETE', '/zones/ff12ab34cd5611334422ab3322997650/'
+                 'dns_records/fc12ab34cd5611334422ab3322997653')
+        ])
+
+    def test_alias(self):
+        provider = CloudflareProvider('test', 'email', 'token')
+
+        # A CNAME for us to transform to ALIAS
+        provider.zone_records = Mock(return_value=[
+            {
+                "id": "fc12ab34cd5611334422ab3322997642",
+                "type": "CNAME",
+                "name": "unit.tests",
+                "content": "www.unit.tests",
+                "proxiable": True,
+                "proxied": False,
+                "ttl": 300,
+                "locked": False,
+                "zone_id": "ff12ab34cd5611334422ab3322997650",
+                "zone_name": "unit.tests",
+                "modified_on": "2017-03-11T18:01:43.420689Z",
+                "created_on": "2017-03-11T18:01:43.420689Z",
+                "meta": {
+                    "auto_added": False
+                }
+            },
+        ])
+
+        zone = Zone('unit.tests.', [])
+        provider.populate(zone)
+        self.assertEquals(1, len(zone.records))
+        record = list(zone.records)[0]
+        self.assertEquals('', record.name)
+        self.assertEquals('unit.tests.', record.fqdn)
+        self.assertEquals('ALIAS', record._type)
+        self.assertEquals('www.unit.tests.', record.value)
+
+        # Make sure we transform back to CNAME going the other way
+        contents = provider._gen_contents(record)
+        self.assertEquals({
+            'content': u'www.unit.tests.',
+            'name': 'unit.tests',
+            'ttl': 300,
+            'type': 'CNAME'
+        }, list(contents)[0])