Merge branch 'master' into normalize-mx-exchange

.git_hooks_pre-commit

@@ -8,4 +8,4 @@ ROOT=`dirname $GIT`
 
 . $ROOT/env/bin/activate
 $ROOT/script/lint
-$ROOT/script/test
+$ROOT/script/coverage

CHANGELOG.md

@@ -1,3 +1,15 @@
+## v0.9.5 - 2019-??-?? - The big one, with all the dynamic stuff
+
+* dynamic record support, essentially a v2 version of geo records with a lot
+  more flexibility and power. Also support dynamic CNAME records.
+* Route53Provider dynamic record support
+* DynProvider dynamic record support
+* SUPPORTS_DYNAMIC is an optional property, defaults to False
+* Route53Provider health checks support disabling latency measurement
+* CloudflareProvider SRV record unpacking fix
+* DNSMadeEasy provider uses supports to avoid blowing up on unknown record
+  types
+
 ## v0.9.4 - 2019-01-28 - The one with a bunch of stuff, before the big one
 
 * A bunch of "dynamic" stuff that'll be detailed in the next release when

README.md

@@ -149,21 +149,21 @@ The above command pulled the existing data out of Route53 and placed the results
 
 ## Supported providers
 
-| Provider | Requirements | Record Support | GeoDNS Support | Notes |
+| Provider | Requirements | Record Support | Dynamic/Geo Support | Notes |
 |--|--|--|--|--|
 | [AzureProvider](/octodns/provider/azuredns.py) | azure-mgmt-dns | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | |
 | [CloudflareProvider](/octodns/provider/cloudflare.py) | | A, AAAA, ALIAS, CAA, CNAME, MX, NS, SPF, SRV, TXT | No | CAA tags restricted |
 | [DigitalOceanProvider](/octodns/provider/digitalocean.py) | | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted |
 | [DnsMadeEasyProvider](/octodns/provider/dnsmadeeasy.py) | | A, AAAA, ALIAS (ANAME), CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | CAA tags restricted |
 | [DnsimpleProvider](/octodns/provider/dnsimple.py) | | All | No | CAA tags restricted |
-| [DynProvider](/octodns/provider/dyn.py) | dyn | All | Yes | |
+| [DynProvider](/octodns/provider/dyn.py) | dyn | All | Both | |
 | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | |
 | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT  | No | |
-| [Ns1Provider](/octodns/provider/ns1.py) | nsone | All | Yes | No health checking for GeoDNS |
+| [Ns1Provider](/octodns/provider/ns1.py) | nsone | All | Partial Geo | No health checking for GeoDNS |
 | [OVH](/octodns/provider/ovh.py) | ovh | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT, DKIM | No | |
 | [PowerDnsProvider](/octodns/provider/powerdns.py) | | All | No | |
 | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No |  |
-| [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | |
+| [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header |
 | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only |
 | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only |
 | [TinyDnsFileSource](/octodns/source/tinydns.py) | | A, CNAME, MX, NS, PTR | No | read-only |

octodns/provider/route53.py

@@ -9,11 +9,13 @@ from boto3 import client
 from botocore.config import Config
 from collections import defaultdict
 from incf.countryutils.transformations import cca_to_ctca2
+from ipaddress import AddressValueError, ip_address
 from uuid import uuid4
 import logging
 import re
 
 from ..record import Record, Update
+from ..record.geo import GeoCodes
 from .base import BaseProvider
 
 
@@ -29,19 +31,105 @@ def _octal_replace(s):
 class _Route53Record(object):
 
     @classmethod
-    def new(self, provider, record, creating):
+    def _new_dynamic(cls, provider, record, hosted_zone_id, creating):
+        # Creates the RRSets that correspond to the given dynamic record
         ret = set()
-        if getattr(record, 'geo', False):
-            ret.add(_Route53GeoDefault(provider, record, creating))
-            for ident, geo in record.geo.items():
-                ret.add(_Route53GeoRecord(provider, record, ident, geo,
-                                          creating))
-        else:
-            ret.add(_Route53Record(provider, record, creating))
+
+        # HostedZoneId wants just the last bit, but the place we're getting
+        # this from looks like /hostedzone/Z424CArX3BB224
+        hosted_zone_id = hosted_zone_id.split('/', 2)[-1]
+
+        # Create the default pool which comes from the base `values` of the
+        # record object. Its only used if all other values fail their
+        # healthchecks, which hopefully never happens.
+        fqdn = record.fqdn
+        ret.add(_Route53Record(provider, record, creating,
+                               '_octodns-default-pool.{}'.format(fqdn)))
+
+        # Pools
+        for pool_name, pool in record.dynamic.pools.items():
+
+            # Create the primary, this will be the rrset that geo targeted
+            # rrsets will point to when they want to use a pool of values. It's
+            # a primary and observes target health so if all the values for
+            # this pool go red, we'll use the fallback/SECONDARY just below
+            ret.add(_Route53DynamicPool(provider, hosted_zone_id, record,
+                                        pool_name, creating))
+
+            # Create the fallback for this pool
+            fallback = pool.data.get('fallback', False)
+            if fallback:
+                # We have an explicitly configured fallback, another pool to
+                # use if all our values go red. This RRSet configures that pool
+                # as the next best option
+                ret.add(_Route53DynamicPool(provider, hosted_zone_id, record,
+                                            pool_name, creating,
+                                            target_name=fallback))
+            else:
+                # We fallback on the default, no explicit fallback so if all of
+                # this pool's values go red we'll fallback to the base
+                # (non-health-checked) default pool of values
+                ret.add(_Route53DynamicPool(provider, hosted_zone_id, record,
+                                            pool_name, creating,
+                                            target_name='default'))
+
+            # Create the values for this pool. These are health checked and in
+            # general each unique value will have an associated healthcheck.
+            # The PRIMARY pool up above will point to these RRSets which will
+            # be served out according to their weights
+            for i, value in enumerate(pool.data['values']):
+                weight = value['weight']
+                value = value['value']
+                ret.add(_Route53DynamicValue(provider, record, pool_name,
+                                             value, weight, i, creating))
+
+        # Rules
+        for i, rule in enumerate(record.dynamic.rules):
+            pool_name = rule.data['pool']
+            geos = rule.data.get('geos', [])
+            if geos:
+                for geo in geos:
+                    # Create a RRSet for each geo in each rule that uses the
+                    # desired target pool
+                    ret.add(_Route53DynamicRule(provider, hosted_zone_id,
+                                                record, pool_name, i,
+                                                creating, geo=geo))
+            else:
+                # There's no geo's for this rule so it's the catchall that will
+                # just point things that don't match any geo rules to the
+                # specified pool
+                ret.add(_Route53DynamicRule(provider, hosted_zone_id, record,
+                                            pool_name, i, creating))
+
         return ret
 
-    def __init__(self, provider, record, creating):
-        self.fqdn = record.fqdn
+    @classmethod
+    def _new_geo(cls, provider, record, creating):
+        # Creates the RRSets that correspond to the given geo record
+        ret = set()
+
+        ret.add(_Route53GeoDefault(provider, record, creating))
+        for ident, geo in record.geo.items():
+            ret.add(_Route53GeoRecord(provider, record, ident, geo,
+                                      creating))
+
+        return ret
+
+    @classmethod
+    def new(cls, provider, record, hosted_zone_id, creating):
+        # Creates the RRSets that correspond to the given record
+
+        if getattr(record, 'dynamic', False):
+            ret = cls._new_dynamic(provider, record, hosted_zone_id, creating)
+            return ret
+        elif getattr(record, 'geo', False):
+            return cls._new_geo(provider, record, creating)
+
+        # Its a simple record that translates into a single RRSet
+        return set((_Route53Record(provider, record, creating),))
+
+    def __init__(self, provider, record, creating, fqdn_override=None):
+        self.fqdn = fqdn_override or record.fqdn
         self._type = record._type
         self.ttl = record.ttl
 
@@ -83,6 +171,15 @@ class _Route53Record(object):
         return '_Route53Record<{} {} {} {}>'.format(self.fqdn, self._type,
                                                     self.ttl, self.values)
 
+    def _value_convert_value(self, value, record):
+        return value
+
+    _value_convert_A = _value_convert_value
+    _value_convert_AAAA = _value_convert_value
+    _value_convert_NS = _value_convert_value
+    _value_convert_CNAME = _value_convert_value
+    _value_convert_PTR = _value_convert_value
+
     def _values_for_values(self, record):
         return record.values
 
@@ -90,9 +187,11 @@ class _Route53Record(object):
     _values_for_AAAA = _values_for_values
     _values_for_NS = _values_for_values
 
+    def _value_convert_CAA(self, value, record):
+        return '{} {} "{}"'.format(value.flags, value.tag, value.value)
+
     def _values_for_CAA(self, record):
-        return ['{} {} "{}"'.format(v.flags, v.tag, v.value)
-                for v in record.values]
+        return [self._value_convert_CAA(v, record) for v in record.values]
 
     def _values_for_value(self, record):
         return [record.value]
@@ -100,18 +199,28 @@ class _Route53Record(object):
     _values_for_CNAME = _values_for_value
     _values_for_PTR = _values_for_value
 
+    def _value_convert_MX(self, value, record):
+        return '{} {}'.format(value.preference, value.exchange)
+
     def _values_for_MX(self, record):
-        return ['{} {}'.format(v.preference, v.exchange)
-                for v in record.values]
+        return [self._value_convert_MX(v, record) for v in record.values]
+
+    def _value_convert_NAPTR(self, value, record):
+        return '{} {} "{}" "{}" "{}" {}' \
+            .format(value.order, value.preference,
+                    value.flags if value.flags else '',
+                    value.service if value.service else '',
+                    value.regexp if value.regexp else '',
+                    value.replacement)
 
     def _values_for_NAPTR(self, record):
-        return ['{} {} "{}" "{}" "{}" {}'
-                .format(v.order, v.preference,
-                        v.flags if v.flags else '',
-                        v.service if v.service else '',
-                        v.regexp if v.regexp else '',
-                        v.replacement)
-                for v in record.values]
+        return [self._value_convert_NAPTR(v, record) for v in record.values]
+
+    def _value_convert_quoted(self, value, record):
+        return record.chunked_value(value)
+
+    _value_convert_SPF = _value_convert_quoted
+    _value_convert_TXT = _value_convert_quoted
 
     def _values_for_quoted(self, record):
         return record.chunked_values
@@ -119,10 +228,178 @@ class _Route53Record(object):
     _values_for_SPF = _values_for_quoted
     _values_for_TXT = _values_for_quoted
 
+    def _value_for_SRV(self, value, record):
+        return '{} {} {} {}'.format(value.priority, value.weight,
+                                    value.port, value.target)
+
     def _values_for_SRV(self, record):
-        return ['{} {} {} {}'.format(v.priority, v.weight, v.port,
-                                     v.target)
-                for v in record.values]
+        return [self._value_for_SRV(v, record) for v in record.values]
+
+
+class _Route53DynamicPool(_Route53Record):
+
+    def __init__(self, provider, hosted_zone_id, record, pool_name, creating,
+                 target_name=None):
+        fqdn_override = '_octodns-{}-pool.{}'.format(pool_name, record.fqdn)
+        super(_Route53DynamicPool, self) \
+            .__init__(provider, record, creating, fqdn_override=fqdn_override)
+
+        self.hosted_zone_id = hosted_zone_id
+        self.pool_name = pool_name
+
+        self.target_name = target_name
+        if target_name:
+            # We're pointing down the chain
+            self.target_dns_name = '_octodns-{}-pool.{}'.format(target_name,
+                                                                record.fqdn)
+        else:
+            # We're a paimary, point at our values
+            self.target_dns_name = '_octodns-{}-value.{}'.format(pool_name,
+                                                                 record.fqdn)
+
+    @property
+    def mode(self):
+        return 'Secondary' if self.target_name else 'Primary'
+
+    @property
+    def identifer(self):
+        if self.target_name:
+            return '{}-{}-{}'.format(self.pool_name, self.mode,
+                                     self.target_name)
+        return '{}-{}'.format(self.pool_name, self.mode)
+
+    def mod(self, action):
+        return {
+            'Action': action,
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': self.target_dns_name,
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': self.hosted_zone_id,
+                },
+                'Failover': 'SECONDARY' if self.target_name else 'PRIMARY',
+                'Name': self.fqdn,
+                'SetIdentifier': self.identifer,
+                'Type': self._type,
+            }
+        }
+
+    def __hash__(self):
+        return '{}:{}:{}'.format(self.fqdn, self._type,
+                                 self.identifer).__hash__()
+
+    def __repr__(self):
+        return '_Route53DynamicPool<{} {} {} {}>' \
+            .format(self.fqdn, self._type, self.mode, self.target_dns_name)
+
+
+class _Route53DynamicRule(_Route53Record):
+
+    def __init__(self, provider, hosted_zone_id, record, pool_name, index,
+                 creating, geo=None):
+        super(_Route53DynamicRule, self).__init__(provider, record, creating)
+
+        self.hosted_zone_id = hosted_zone_id
+        self.geo = geo
+        self.pool_name = pool_name
+        self.index = index
+
+        self.target_dns_name = '_octodns-{}-pool.{}'.format(pool_name,
+                                                            record.fqdn)
+
+    @property
+    def identifer(self):
+        return '{}-{}-{}'.format(self.index, self.pool_name, self.geo)
+
+    def mod(self, action):
+        rrset = {
+            'AliasTarget': {
+                'DNSName': self.target_dns_name,
+                'EvaluateTargetHealth': True,
+                'HostedZoneId': self.hosted_zone_id,
+            },
+            'GeoLocation': {
+                'CountryCode': '*'
+            },
+            'Name': self.fqdn,
+            'SetIdentifier': self.identifer,
+            'Type': self._type,
+        }
+
+        if self.geo:
+            geo = GeoCodes.parse(self.geo)
+
+            if geo['province_code']:
+                rrset['GeoLocation'] = {
+                    'CountryCode': geo['country_code'],
+                    'SubdivisionCode': geo['province_code'],
+                }
+            elif geo['country_code']:
+                rrset['GeoLocation'] = {
+                    'CountryCode': geo['country_code']
+                }
+            else:
+                rrset['GeoLocation'] = {
+                    'ContinentCode': geo['continent_code'],
+                }
+
+        return {
+            'Action': action,
+            'ResourceRecordSet': rrset,
+        }
+
+    def __hash__(self):
+        return '{}:{}:{}'.format(self.fqdn, self._type,
+                                 self.identifer).__hash__()
+
+    def __repr__(self):
+        return '_Route53DynamicRule<{} {} {} {} {}>' \
+            .format(self.fqdn, self._type, self.index, self.geo,
+                    self.target_dns_name)
+
+
+class _Route53DynamicValue(_Route53Record):
+
+    def __init__(self, provider, record, pool_name, value, weight, index,
+                 creating):
+        fqdn_override = '_octodns-{}-value.{}'.format(pool_name, record.fqdn)
+        super(_Route53DynamicValue, self).__init__(provider, record, creating,
+                                                   fqdn_override=fqdn_override)
+
+        self.pool_name = pool_name
+        self.index = index
+        value_convert = getattr(self, '_value_convert_{}'.format(record._type))
+        self.value = value_convert(value, record)
+        self.weight = weight
+
+        self.health_check_id = provider.get_health_check_id(record, self.value,
+                                                            creating)
+
+    @property
+    def identifer(self):
+        return '{}-{:03d}'.format(self.pool_name, self.index)
+
+    def mod(self, action):
+        return {
+            'Action': action,
+            'ResourceRecordSet': {
+                'HealthCheckId': self.health_check_id,
+                'Name': self.fqdn,
+                'ResourceRecords': [{'Value': self.value}],
+                'SetIdentifier': self.identifer,
+                'TTL': self.ttl,
+                'Type': self._type,
+                'Weight': self.weight,
+            }
+        }
+
+    def __hash__(self):
+        return '{}:{}:{}'.format(self.fqdn, self._type,
+                                 self.identifer).__hash__()
+
+    def __repr__(self):
+        return '_Route53DynamicValue<{} {} {} {}>' \
+            .format(self.fqdn, self._type, self.identifer, self.value)
 
 
 class _Route53GeoDefault(_Route53Record):
@@ -156,8 +433,9 @@ class _Route53GeoRecord(_Route53Record):
         super(_Route53GeoRecord, self).__init__(provider, record, creating)
         self.geo = geo
 
-        self.health_check_id = provider.get_health_check_id(record, ident,
-                                                            geo, creating)
+        value = geo.values[0]
+        self.health_check_id = provider.get_health_check_id(record, value,
+                                                            creating)
 
     def mod(self, action):
         geo = self.geo
@@ -211,6 +489,44 @@ class _Route53GeoRecord(_Route53Record):
                                                           self.values)
 
 
+_mod_keyer_action_order = {
+    'DELETE': 0,  # Delete things first
+    'CREATE': 1,  # Then Create things
+    'UPSERT': 2,  # Upsert things last
+}
+
+
+def _mod_keyer(mod):
+    rrset = mod['ResourceRecordSet']
+    action_order = _mod_keyer_action_order[mod['Action']]
+
+    # We're sorting by 3 "columns", the action, the rrset type, and finally the
+    # name/id of the rrset. This ensures that Route53 won't see a RRSet that
+    # targets another that hasn't been seen yet. I.e. targets must come before
+    # things that target them. We sort on types of things rather than
+    # explicitly looking for targeting relationships since that's sufficent and
+    # easier to grok/do.
+
+    if rrset.get('GeoLocation', False):
+        return (action_order, 3, rrset['SetIdentifier'])
+    elif rrset.get('AliasTarget', False):
+        # We use an alias
+        if rrset.get('Failover', False) == 'SECONDARY':
+            # We're a secondary we'll ref primaries
+            return (action_order, 2, rrset['Name'])
+        else:
+            # We're a primary we'll ref values
+            return (action_order, 1, rrset['Name'])
+
+    # We're just a plain value, these come first
+    return (action_order, 0, rrset['Name'])
+
+
+def _parse_pool_name(n):
+    # Parse the pool name out of _octodns-<pool-name>-pool...
+    return n.split('.', 1)[0][9:-5]
+
+
 class Route53Provider(BaseProvider):
     '''
     AWS Route53 Provider
@@ -232,8 +548,7 @@ class Route53Provider(BaseProvider):
     In general the account used will need full permissions on Route53.
     '''
     SUPPORTS_GEO = True
-    # TODO: dynamic
-    SUPPORTS_DYNAMIC = False
+    SUPPORTS_DYNAMIC = True
     SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR',
                     'SPF', 'SRV', 'TXT'))
 
@@ -465,6 +780,79 @@ class Route53Provider(BaseProvider):
 
         return self._r53_rrsets[zone_id]
 
+    def _data_for_dynamic(self, name, _type, rrsets):
+        # This converts a bunch of RRSets into their corresponding dynamic
+        # Record. It's used by populate.
+        pools = defaultdict(lambda: {'values': []})
+        # Data to build our rules will be collected here and "converted" into
+        # their final form below
+        rules = defaultdict(lambda: {'pool': None, 'geos': []})
+        # Base/empty data
+        data = {
+            'dynamic': {
+                'pools': pools,
+                'rules': [],
+            }
+        }
+
+        # For all the rrsets that comprise this dynamic record
+        for rrset in rrsets:
+            name = rrset['Name']
+            if '-pool.' in name:
+                # This is a pool rrset
+                pool_name = _parse_pool_name(name)
+                if pool_name == 'default':
+                    # default becomes the base for the record and its
+                    # value(s) will fill the non-dynamic values
+                    data_for = getattr(self, '_data_for_{}'.format(_type))
+                    data.update(data_for(rrset))
+                elif rrset['Failover'] == 'SECONDARY':
+                    # This is a failover record, we'll ignore PRIMARY, but
+                    # SECONDARY will tell us what the pool's fallback is
+                    fallback_name = \
+                        _parse_pool_name(rrset['AliasTarget']['DNSName'])
+                    # Don't care about default fallbacks, anything else
+                    # we'll record
+                    if fallback_name != 'default':
+                        pools[pool_name]['fallback'] = fallback_name
+            elif 'GeoLocation' in rrset:
+                # These are rules
+                _id = rrset['SetIdentifier']
+                # We record rule index as the first part of set-id, the 2nd
+                # part just ensures uniqueness across geos and is ignored
+                i = int(_id.split('-', 1)[0])
+                target_pool = _parse_pool_name(rrset['AliasTarget']['DNSName'])
+                # Record the pool
+                rules[i]['pool'] = target_pool
+                # Record geo if we have one
+                geo = self._parse_geo(rrset)
+                if geo:
+                    rules[i]['geos'].append(geo)
+            else:
+                # These are the pool value(s)
+                # Grab the pool name out of the SetIdentifier, format looks
+                # like ...-000 where 000 is a zero-padded index for the value
+                # it's ignored only used to make sure the value is unique
+                pool_name = rrset['SetIdentifier'][:-4]
+                value = rrset['ResourceRecords'][0]['Value']
+                pools[pool_name]['values'].append({
+                    'value': value,
+                    'weight': rrset['Weight'],
+                })
+
+        # Convert our map of rules into an ordered list now that we have all
+        # the data
+        for _, rule in sorted(rules.items()):
+            r = {
+                'pool': rule['pool'],
+            }
+            geos = sorted(rule['geos'])
+            if geos:
+                r['geos'] = geos
+            data['dynamic']['rules'].append(r)
+
+        return data
+
     def populate(self, zone, target=False, lenient=False):
         self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
                        target, lenient)
@@ -476,21 +864,46 @@ class Route53Provider(BaseProvider):
         if zone_id:
             exists = True
             records = defaultdict(lambda: defaultdict(list))
+            dynamic = defaultdict(lambda: defaultdict(list))
+
             for rrset in self._load_records(zone_id):
                 record_name = zone.hostname_from_fqdn(rrset['Name'])
                 record_name = _octal_replace(record_name)
                 record_type = rrset['Type']
                 if record_type not in self.SUPPORTS:
+                    # Skip stuff we don't support
                     continue
-                if 'AliasTarget' in rrset:
-                    # Alias records are Route53 specific and are not
-                    # portable, so we need to skip them
-                    self.log.warning("%s is an Alias record. Skipping..."
-                                     % rrset['Name'])
+                if record_name.startswith('_octodns-'):
+                    # Part of a dynamic record
+                    try:
+                        record_name = record_name.split('.', 1)[1]
+                    except IndexError:
+                        record_name = ''
+                    dynamic[record_name][record_type].append(rrset)
                     continue
+                elif 'AliasTarget' in rrset:
+                    if rrset['AliasTarget']['DNSName'].startswith('_octodns-'):
+                        # Part of a dynamic record
+                        dynamic[record_name][record_type].append(rrset)
+                    else:
+                        # Alias records are Route53 specific and are not
+                        # portable, so we need to skip them
+                        self.log.warning("%s is an Alias record. Skipping..."
+                                         % rrset['Name'])
+                    continue
+                # A basic record (potentially including geo)
                 data = getattr(self, '_data_for_{}'.format(record_type))(rrset)
                 records[record_name][record_type].append(data)
 
+            # Convert the dynamic rrsets to Records
+            for name, types in dynamic.items():
+                for _type, rrsets in types.items():
+                    data = self._data_for_dynamic(name, _type, rrsets)
+                    record = Record.new(zone, name, data, source=self,
+                                        lenient=lenient)
+                    zone.add_record(record, lenient=lenient)
+
+            # Convert the basic (potentially with geo) rrsets to records
             for name, types in records.items():
                 for _type, data in types.items():
                     if len(data) > 1:
@@ -537,6 +950,7 @@ class Route53Provider(BaseProvider):
                         # ignore anything else
                         continue
                     checks[health_check['Id']] = health_check
+
                 more = resp['IsTruncated']
                 start['Marker'] = resp.get('NextMarker', None)
 
@@ -551,27 +965,44 @@ class Route53Provider(BaseProvider):
             .get('measure_latency', True)
 
     def _health_check_equivilent(self, host, path, protocol, port,
-                                 measure_latency, health_check,
-                                 first_value=None):
+                                 measure_latency, health_check, value=None):
         config = health_check['HealthCheckConfig']
+
+        # So interestingly Route53 normalizes IPAddress which will cause us to
+        # fail to find see things as equivalent. To work around this we'll
+        # ip_address's returned object for equivalence
+        # E.g 2001:4860:4860::8842 -> 2001:4860:4860:0:0:0:0:8842
+        if value:
+            value = ip_address(unicode(value))
+            config_ip_address = ip_address(unicode(config['IPAddress']))
+        else:
+            # No value so give this a None to match value's
+            config_ip_address = None
+
         return host == config['FullyQualifiedDomainName'] and \
             path == config['ResourcePath'] and protocol == config['Type'] \
             and port == config['Port'] and \
             measure_latency == config['MeasureLatency'] and \
-            (first_value is None or first_value == config['IPAddress'])
+            value == config_ip_address
 
-    def get_health_check_id(self, record, ident, geo, create):
+    def get_health_check_id(self, record, value, create):
         # fqdn & the first value are special, we use them to match up health
         # checks to their records. Route53 health checks check a single ip and
         # we're going to assume that ips are interchangeable to avoid
         # health-checking each one independently
         fqdn = record.fqdn
-        first_value = geo.values[0]
-        self.log.debug('get_health_check_id: fqdn=%s, type=%s, geo=%s, '
-                       'first_value=%s', fqdn, record._type, ident,
-                       first_value)
+        self.log.debug('get_health_check_id: fqdn=%s, type=%s, value=%s',
+                       fqdn, record._type, value)
+
+        try:
+            ip_address(unicode(value))
+            # We're working with an IP, host is the Host header
+            healthcheck_host = record.healthcheck_host
+        except (AddressValueError, ValueError):
+            # This isn't an IP, host is the value, value should be None
+            healthcheck_host = value
+            value = None
 
-        healthcheck_host = record.healthcheck_host
         healthcheck_path = record.healthcheck_path
         healthcheck_protocol = record.healthcheck_protocol
         healthcheck_port = record.healthcheck_port
@@ -591,7 +1022,7 @@ class Route53Provider(BaseProvider):
                                              healthcheck_port,
                                              healthcheck_latency,
                                              health_check,
-                                             first_value=first_value):
+                                             value=value):
                 # this is the health check we're looking for
                 self.log.debug('get_health_check_id:   found match id=%s', id)
                 return id
@@ -606,28 +1037,44 @@ class Route53Provider(BaseProvider):
             'EnableSNI': healthcheck_protocol == 'HTTPS',
             'FailureThreshold': 6,
             'FullyQualifiedDomainName': healthcheck_host,
-            'IPAddress': first_value,
             'MeasureLatency': healthcheck_latency,
             'Port': healthcheck_port,
             'RequestInterval': 10,
             'ResourcePath': healthcheck_path,
             'Type': healthcheck_protocol,
         }
+        if value:
+            config['IPAddress'] = value
+
         ref = '{}:{}:{}:{}'.format(self.HEALTH_CHECK_VERSION, record._type,
                                    record.fqdn, uuid4().hex[:12])
         resp = self._conn.create_health_check(CallerReference=ref,
                                               HealthCheckConfig=config)
         health_check = resp['HealthCheck']
         id = health_check['Id']
+
+        # Set a Name for the benefit of the UI
+        name = '{}:{} - {}'.format(record.fqdn, record._type,
+                                   value or healthcheck_host)
+        self._conn.change_tags_for_resource(ResourceType='healthcheck',
+                                            ResourceId=id,
+                                            AddTags=[{
+                                                'Key': 'Name',
+                                                'Value': name,
+                                            }])
+        # Manually add it to our cache
+        health_check['Tags'] = {
+            'Name': name
+        }
+
         # store the new health check so that we'll be able to find it in the
         # future
         self._health_checks[id] = health_check
         self.log.info('get_health_check_id: created id=%s, host=%s, '
                       'path=%s, protocol=%s, port=%d, measure_latency=%r, '
-                      'first_value=%s',
-                      id, healthcheck_host, healthcheck_path,
+                      'value=%s', id, healthcheck_host, healthcheck_path,
                       healthcheck_protocol, healthcheck_port,
-                      healthcheck_latency, first_value)
+                      healthcheck_latency, value)
         return id
 
     def _gc_health_checks(self, record, new):
@@ -664,25 +1111,26 @@ class Route53Provider(BaseProvider):
                                   id)
                     self._conn.delete_health_check(HealthCheckId=id)
 
-    def _gen_records(self, record, creating=False):
+    def _gen_records(self, record, zone_id, creating=False):
         '''
         Turns an octodns.Record into one or more `_Route53*`s
         '''
-        return _Route53Record.new(self, record, creating)
+        return _Route53Record.new(self, record, zone_id, creating)
 
-    def _mod_Create(self, change):
+    def _mod_Create(self, change, zone_id):
         # New is the stuff that needs to be created
-        new_records = self._gen_records(change.new, creating=True)
+        new_records = self._gen_records(change.new, zone_id, creating=True)
         # Now is a good time to clear out any unused health checks since we
         # know what we'll be using going forward
         self._gc_health_checks(change.new, new_records)
         return self._gen_mods('CREATE', new_records)
 
-    def _mod_Update(self, change):
+    def _mod_Update(self, change, zone_id):
         # See comments in _Route53Record for how the set math is made to do our
         # bidding here.
-        existing_records = self._gen_records(change.existing, creating=False)
-        new_records = self._gen_records(change.new, creating=True)
+        existing_records = self._gen_records(change.existing, zone_id,
+                                             creating=False)
+        new_records = self._gen_records(change.new, zone_id, creating=True)
         # Now is a good time to clear out any unused health checks since we
         # know what we'll be using going forward
         self._gc_health_checks(change.new, new_records)
@@ -704,14 +1152,91 @@ class Route53Provider(BaseProvider):
             self._gen_mods('CREATE', creates) + \
             self._gen_mods('UPSERT', upserts)
 
-    def _mod_Delete(self, change):
+    def _mod_Delete(self, change, zone_id):
         # Existing is the thing that needs to be deleted
-        existing_records = self._gen_records(change.existing, creating=False)
+        existing_records = self._gen_records(change.existing, zone_id,
+                                             creating=False)
         # Now is a good time to clear out all the health checks since we know
         # we're done with them
         self._gc_health_checks(change.existing, [])
         return self._gen_mods('DELETE', existing_records)
 
+    def _extra_changes_update_needed(self, record, rrset):
+        healthcheck_host = record.healthcheck_host
+        healthcheck_path = record.healthcheck_path
+        healthcheck_protocol = record.healthcheck_protocol
+        healthcheck_port = record.healthcheck_port
+        healthcheck_latency = self._healthcheck_measure_latency(record)
+
+        try:
+            health_check_id = rrset['HealthCheckId']
+            health_check = self.health_checks[health_check_id]
+            caller_ref = health_check['CallerReference']
+            if caller_ref.startswith(self.HEALTH_CHECK_VERSION):
+                if self._health_check_equivilent(healthcheck_host,
+                                                 healthcheck_path,
+                                                 healthcheck_protocol,
+                                                 healthcheck_port,
+                                                 healthcheck_latency,
+                                                 health_check):
+                    # it has the right health check
+                    return False
+        except (IndexError, KeyError):
+            # no health check id or one that isn't the right version
+            pass
+
+        # no good, doesn't have the right health check, needs an update
+        self.log.info('_extra_changes_update_needed: health-check caused '
+                      'update of %s:%s', record.fqdn, record._type)
+        return True
+
+    def _extra_changes_geo_needs_update(self, zone_id, record):
+        # OK this is a record we don't have change for that does have geo
+        # information. We need to look and see if it needs to be updated b/c of
+        # a health check version bump or other mismatch
+        self.log.debug('_extra_changes_geo_needs_update: inspecting=%s, %s',
+                       record.fqdn, record._type)
+
+        fqdn = record.fqdn
+
+        # loop through all the r53 rrsets
+        for rrset in self._load_records(zone_id):
+            if fqdn == rrset['Name'] and record._type == rrset['Type'] and \
+               rrset.get('GeoLocation', {}).get('CountryCode', False) != '*' \
+               and self._extra_changes_update_needed(record, rrset):
+                # no good, doesn't have the right health check, needs an update
+                self.log.info('_extra_changes_geo_needs_update: health-check '
+                              'caused update of %s:%s', record.fqdn,
+                              record._type)
+                return True
+
+        return False
+
+    def _extra_changes_dynamic_needs_update(self, zone_id, record):
+        # OK this is a record we don't have change for that does have dynamic
+        # information. We need to look and see if it needs to be updated b/c of
+        # a health check version bump or other mismatch
+        self.log.debug('_extra_changes_dynamic_needs_update: inspecting=%s, '
+                       '%s', record.fqdn, record._type)
+
+        fqdn = record.fqdn
+
+        # loop through all the r53 rrsets
+        for rrset in self._load_records(zone_id):
+            name = rrset['Name']
+
+            if record._type == rrset['Type'] and name.endswith(fqdn) and \
+               name.startswith('_octodns-') and '-value.' in name and \
+               '-default-' not in name and \
+               self._extra_changes_update_needed(record, rrset):
+                # no good, doesn't have the right health check, needs an update
+                self.log.info('_extra_changes_dynamic_needs_update: '
+                              'health-check caused update of %s:%s',
+                              record.fqdn, record._type)
+                return True
+
+        return False
+
     def _extra_changes(self, desired, changes, **kwargs):
         self.log.debug('_extra_changes: desired=%s', desired.name)
         zone_id = self._get_zone_id(desired.name)
@@ -722,61 +1247,20 @@ class Route53Provider(BaseProvider):
         changed = set([c.record for c in changes])
         # ok, now it's time for the reason we're here, we need to go over all
         # the desired records
-        extra = []
+        extras = []
         for record in desired.records:
             if record in changed:
                 # already have a change for it, skipping
                 continue
-            if not getattr(record, 'geo', False):
-                # record doesn't support geo, we don't need to inspect it
-                continue
-            # OK this is a record we don't have change for that does have geo
-            # information. We need to look and see if it needs to be updated
-            # b/c of a health check version bump
-            self.log.debug('_extra_changes:   inspecting=%s, %s', record.fqdn,
-                           record._type)
 
-            healthcheck_host = record.healthcheck_host
-            healthcheck_path = record.healthcheck_path
-            healthcheck_protocol = record.healthcheck_protocol
-            healthcheck_port = record.healthcheck_port
-            healthcheck_latency = self._healthcheck_measure_latency(record)
-            fqdn = record.fqdn
+            if getattr(record, 'geo', False):
+                if self._extra_changes_geo_needs_update(zone_id, record):
+                    extras.append(Update(record, record))
+            elif getattr(record, 'dynamic', False):
+                if self._extra_changes_dynamic_needs_update(zone_id, record):
+                    extras.append(Update(record, record))
 
-            # loop through all the r53 rrsets
-            for rrset in self._load_records(zone_id):
-                if fqdn != rrset['Name'] or record._type != rrset['Type']:
-                    # not a name and type match
-                    continue
-                if rrset.get('GeoLocation', {}) \
-                   .get('CountryCode', False) == '*':
-                    # it's a default record
-                    continue
-                # we expect a healthcheck now
-                try:
-                    health_check_id = rrset['HealthCheckId']
-                    health_check = self.health_checks[health_check_id]
-                    caller_ref = health_check['CallerReference']
-                    if caller_ref.startswith(self.HEALTH_CHECK_VERSION):
-                        if self._health_check_equivilent(healthcheck_host,
-                                                         healthcheck_path,
-                                                         healthcheck_protocol,
-                                                         healthcheck_port,
-                                                         healthcheck_latency,
-                                                         health_check):
-                            # it has the right health check
-                            continue
-                except (IndexError, KeyError):
-                    # no health check id or one that isn't the right version
-                    pass
-                # no good, doesn't have the right health check, needs an update
-                self.log.info('_extra_changes:     health-check caused '
-                              'update of %s:%s', record.fqdn, record._type)
-                extra.append(Update(record, record))
-                # We don't need to process this record any longer
-                break
-
-        return extra
+        return extras
 
     def _apply(self, plan):
         desired = plan.desired
@@ -788,9 +1272,17 @@ class Route53Provider(BaseProvider):
         batch_rs_count = 0
         zone_id = self._get_zone_id(desired.name, True)
         for c in changes:
-            mods = getattr(self, '_mod_{}'.format(c.__class__.__name__))(c)
+            # Generate the mods for this change
+            mod_type = getattr(self, '_mod_{}'.format(c.__class__.__name__))
+            mods = mod_type(c, zone_id)
+
+            # Order our mods to make sure targets exist before alises point to
+            # them and we CRUD in the desired order
+            mods.sort(key=_mod_keyer)
+
             mods_rs_count = sum(
-                [len(m['ResourceRecordSet']['ResourceRecords']) for m in mods]
+                [len(m['ResourceRecordSet'].get('ResourceRecords', ''))
+                 for m in mods]
             )
 
             if mods_rs_count > self.max_changes:

octodns/record/__init__.py

@@ -1067,15 +1067,18 @@ class _ChunkedValuesMixin(_ValuesMixin):
     CHUNK_SIZE = 255
     _unescaped_semicolon_re = re.compile(r'\w;')
 
+    def chunked_value(self, value):
+        value = value.replace('"', '\\"')
+        vs = [value[i:i + self.CHUNK_SIZE]
+              for i in range(0, len(value), self.CHUNK_SIZE)]
+        vs = '" "'.join(vs)
+        return '"{}"'.format(vs)
+
     @property
     def chunked_values(self):
         values = []
         for v in self.values:
-            v = v.replace('"', '\\"')
-            vs = [v[i:i + self.CHUNK_SIZE]
-                  for i in range(0, len(v), self.CHUNK_SIZE)]
-            vs = '" "'.join(vs)
-            values.append('"{}"'.format(vs))
+            values.append(self.chunked_value(v))
         return values
 
 

tests/test_octodns_provider_route53.py

@@ -12,7 +12,7 @@ from mock import patch
 
 from octodns.record import Create, Delete, Record, Update
 from octodns.provider.route53 import Route53Provider, _Route53GeoDefault, \
-    _Route53GeoRecord, _Route53Record, _octal_replace
+    _Route53GeoRecord, _Route53Record, _mod_keyer, _octal_replace
 from octodns.zone import Zone
 
 from helpers import GeoProvider
@@ -42,6 +42,202 @@ class TestOctalReplace(TestCase):
             self.assertEquals(expected, _octal_replace(s))
 
 
+dynamic_rrsets = [{
+    'Name': '_octodns-default-pool.unit.tests.',
+    'ResourceRecords': [{'Value': '1.1.2.1'},
+                        {'Value': '1.1.2.2'}],
+    'TTL': 60,
+    'Type': 'A',
+}, {
+    'HealthCheckId': '76',
+    'Name': '_octodns-ap-southeast-1-value.unit.tests.',
+    'ResourceRecords': [{'Value': '1.4.1.1'}],
+    'SetIdentifier': 'ap-southeast-1-000',
+    'TTL': 60,
+    'Type': 'A',
+    'Weight': 2
+}, {
+    'HealthCheckId': '09',
+    'Name': '_octodns-ap-southeast-1-value.unit.tests.',
+    'ResourceRecords': [{'Value': '1.4.1.2'}],
+    'SetIdentifier': 'ap-southeast-1-001',
+    'TTL': 60,
+    'Type': 'A',
+    'Weight': 2
+}, {
+    'HealthCheckId': 'ab',
+    'Name': '_octodns-eu-central-1-value.unit.tests.',
+    'ResourceRecords': [{'Value': '1.3.1.1'}],
+    'SetIdentifier': 'eu-central-1-000',
+    'TTL': 60,
+    'Type': 'A',
+    'Weight': 1
+}, {
+    'HealthCheckId': '1e',
+    'Name': '_octodns-eu-central-1-value.unit.tests.',
+    'ResourceRecords': [{'Value': '1.3.1.2'}],
+    'SetIdentifier': 'eu-central-1-001',
+    'TTL': 60,
+    'Type': 'A',
+    'Weight': 1
+}, {
+    'HealthCheckId': '2a',
+    'Name': '_octodns-us-east-1-value.unit.tests.',
+    'ResourceRecords': [{'Value': '1.5.1.1'}],
+    'SetIdentifier': 'us-east-1-000',
+    'TTL': 60,
+    'Type': 'A',
+    'Weight': 1
+}, {
+    'HealthCheckId': '61',
+    'Name': '_octodns-us-east-1-value.unit.tests.',
+    'ResourceRecords': [{'Value': '1.5.1.2'}],
+    'SetIdentifier': 'us-east-1-001',
+    'TTL': 60,
+    'Type': 'A',
+    'Weight': 1,
+}, {
+    'AliasTarget': {'DNSName': '_octodns-default-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'Z2'},
+    'Failover': 'SECONDARY',
+    'Name': '_octodns-us-east-1-pool.unit.tests.',
+    'SetIdentifier': 'us-east-1-Secondary-default',
+    'Type': 'A'
+}, {
+    'AliasTarget': {
+        'DNSName': '_octodns-us-east-1-value.unit.tests.',
+        'EvaluateTargetHealth': True,
+        'HostedZoneId': 'Z2'
+    },
+    'Failover': 'PRIMARY',
+    'Name': '_octodns-us-east-1-pool.unit.tests.',
+    'SetIdentifier': 'us-east-1-Primary',
+    'Type': 'A',
+}, {
+    'AliasTarget': {'DNSName': '_octodns-us-east-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'Z2'},
+    'Failover': 'SECONDARY',
+    'Name': '_octodns-eu-central-1-pool.unit.tests.',
+    'SetIdentifier': 'eu-central-1-Secondary-default',
+    'Type': 'A'
+}, {
+    'AliasTarget': {
+        'DNSName': '_octodns-eu-central-1-value.unit.tests.',
+        'EvaluateTargetHealth': True,
+        'HostedZoneId': 'Z2'
+    },
+    'Failover': 'PRIMARY',
+    'Name': '_octodns-eu-central-1-pool.unit.tests.',
+    'SetIdentifier': 'eu-central-1-Primary',
+    'Type': 'A',
+}, {
+    'AliasTarget': {'DNSName': '_octodns-us-east-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'Z2'},
+    'Failover': 'SECONDARY',
+    'Name': '_octodns-ap-southeast-1-pool.unit.tests.',
+    'SetIdentifier': 'ap-southeast-1-Secondary-default',
+    'Type': 'A'
+}, {
+    'AliasTarget': {
+        'DNSName': '_octodns-ap-southeast-1-value.unit.tests.',
+        'EvaluateTargetHealth': True,
+        'HostedZoneId': 'Z2'
+    },
+    'Failover': 'PRIMARY',
+    'Name': '_octodns-ap-southeast-1-pool.unit.tests.',
+    'SetIdentifier': 'ap-southeast-1-Primary',
+    'Type': 'A',
+}, {
+    'AliasTarget': {'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'Z2'},
+    'GeoLocation': {'CountryCode': 'JP'},
+    'Name': 'unit.tests.',
+    'SetIdentifier': '1-ap-southeast-1-AS-JP',
+    'Type': 'A',
+}, {
+    'AliasTarget': {'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'Z2'},
+    'GeoLocation': {'CountryCode': 'CN'},
+    'Name': 'unit.tests.',
+    'SetIdentifier': '1-ap-southeast-1-AS-CN',
+    'Type': 'A',
+}, {
+    'AliasTarget': {'DNSName': '_octodns-eu-central-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'Z2'},
+    'GeoLocation': {'ContinentCode': 'NA-US-FL'},
+    'Name': 'unit.tests.',
+    'SetIdentifier': '2-eu-central-1-NA-US-FL',
+    'Type': 'A',
+}, {
+    'AliasTarget': {'DNSName': '_octodns-eu-central-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'Z2'},
+    'GeoLocation': {'ContinentCode': 'EU'},
+    'Name': 'unit.tests.',
+    'SetIdentifier': '2-eu-central-1-EU',
+    'Type': 'A',
+}, {
+    'AliasTarget': {'DNSName': '_octodns-us-east-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'Z2'},
+    'GeoLocation': {'CountryCode': '*'},
+    'Name': 'unit.tests.',
+    'SetIdentifier': '3-us-east-1-None',
+    'Type': 'A',
+}]
+
+dynamic_record_data = {
+    'dynamic': {
+        'pools': {
+            'ap-southeast-1': {
+                'fallback': 'us-east-1',
+                'values': [{
+                    'weight': 2, 'value': '1.4.1.1'
+                }, {
+                    'weight': 2, 'value': '1.4.1.2'
+                }]
+            },
+            'eu-central-1': {
+                'fallback': 'us-east-1',
+                'values': [{
+                    'weight': 1, 'value': '1.3.1.1'
+                }, {
+                    'weight': 1, 'value': '1.3.1.2'
+                }],
+            },
+            'us-east-1': {
+                'values': [{
+                    'weight': 1, 'value': '1.5.1.1'
+                }, {
+                    'weight': 1, 'value': '1.5.1.2'
+                }],
+            }
+        },
+        'rules': [{
+            'geos': ['AS-CN', 'AS-JP'],
+            'pool': 'ap-southeast-1',
+        }, {
+            'geos': ['EU', 'NA-US-FL'],
+            'pool': 'eu-central-1',
+        }, {
+            'pool': 'us-east-1',
+        }],
+    },
+    'ttl': 60,
+    'type': 'A',
+    'values': [
+        '1.1.2.1',
+        '1.1.2.2',
+    ],
+}
+
+
 class TestRoute53Provider(TestCase):
     expected = Zone('unit.tests.', [])
     for name, data in (
@@ -526,17 +722,6 @@ class TestRoute53Provider(TestCase):
                         'TTL': 61,
                         'Type': 'A'
                     }
-                }, {
-                    'Action': 'UPSERT',
-                    'ResourceRecordSet': {
-                        'GeoLocation': {'CountryCode': '*'},
-                        'Name': 'unit.tests.',
-                        'ResourceRecords': [{'Value': '2.2.3.4'},
-                                            {'Value': '3.2.3.4'}],
-                        'SetIdentifier': 'default',
-                        'TTL': 61,
-                        'Type': 'A'
-                    }
                 }, {
                     'Action': 'UPSERT',
                     'ResourceRecordSet': {
@@ -549,6 +734,17 @@ class TestRoute53Provider(TestCase):
                         'TTL': 61,
                         'Type': 'A'
                     }
+                }, {
+                    'Action': 'UPSERT',
+                    'ResourceRecordSet': {
+                        'GeoLocation': {'CountryCode': '*'},
+                        'Name': 'unit.tests.',
+                        'ResourceRecords': [{'Value': '2.2.3.4'},
+                                            {'Value': '3.2.3.4'}],
+                        'SetIdentifier': 'default',
+                        'TTL': 61,
+                        'Type': 'A'
+                    }
                 }],
                 'Comment': ANY
             },
@@ -586,16 +782,6 @@ class TestRoute53Provider(TestCase):
         change_resource_record_sets_params = {
             'ChangeBatch': {
                 'Changes': [{
-                    'Action': 'DELETE',
-                    'ResourceRecordSet': {
-                        'GeoLocation': {'CountryCode': '*'},
-                        'Name': 'simple.unit.tests.',
-                        'ResourceRecords': [{'Value': '1.2.3.4'},
-                                            {'Value': '2.2.3.4'}],
-                        'SetIdentifier': 'default',
-                        'TTL': 61,
-                        'Type': 'A'}
-                }, {
                     'Action': 'DELETE',
                     'ResourceRecordSet': {
                         'GeoLocation': {'ContinentCode': 'OC'},
@@ -605,6 +791,16 @@ class TestRoute53Provider(TestCase):
                         'SetIdentifier': 'OC',
                         'TTL': 61,
                         'Type': 'A'}
+                }, {
+                    'Action': 'DELETE',
+                    'ResourceRecordSet': {
+                        'GeoLocation': {'CountryCode': '*'},
+                        'Name': 'simple.unit.tests.',
+                        'ResourceRecords': [{'Value': '1.2.3.4'},
+                                            {'Value': '2.2.3.4'}],
+                        'SetIdentifier': 'default',
+                        'TTL': 61,
+                        'Type': 'A'}
                 }, {
                     'Action': 'CREATE',
                     'ResourceRecordSet': {
@@ -773,7 +969,8 @@ class TestRoute53Provider(TestCase):
                 'AF': ['4.2.3.4'],
             }
         })
-        id = provider.get_health_check_id(record, 'AF', record.geo['AF'], True)
+        value = record.geo['AF'].values[0]
+        id = provider.get_health_check_id(record, value, True)
         self.assertEquals('42', id)
 
     def test_health_check_create(self):
@@ -840,6 +1037,7 @@ class TestRoute53Provider(TestCase):
             'CallerReference': ANY,
             'HealthCheckConfig': health_check_config,
         })
+        stubber.add_response('change_tags_for_resource', {})
 
         record = Record.new(self.expected, '', {
             'ttl': 61,
@@ -859,12 +1057,42 @@ class TestRoute53Provider(TestCase):
         })
 
         # if not allowed to create returns none
-        id = provider.get_health_check_id(record, 'AF', record.geo['AF'],
-                                          False)
+        value = record.geo['AF'].values[0]
+        id = provider.get_health_check_id(record, value, False)
         self.assertFalse(id)
 
         # when allowed to create we do
-        id = provider.get_health_check_id(record, 'AF', record.geo['AF'], True)
+        id = provider.get_health_check_id(record, value, True)
+        self.assertEquals('42', id)
+        stubber.assert_no_pending_responses()
+
+        # A CNAME style healthcheck, without a value
+
+        health_check_config = {
+            'EnableSNI': False,
+            'FailureThreshold': 6,
+            'FullyQualifiedDomainName': 'target-1.unit.tests.',
+            'MeasureLatency': True,
+            'Port': 8080,
+            'RequestInterval': 10,
+            'ResourcePath': '/_status',
+            'Type': 'HTTP'
+        }
+        stubber.add_response('create_health_check', {
+            'HealthCheck': {
+                'Id': '42',
+                'CallerReference': self.caller_ref,
+                'HealthCheckConfig': health_check_config,
+                'HealthCheckVersion': 1,
+            },
+            'Location': 'http://url',
+        }, {
+            'CallerReference': ANY,
+            'HealthCheckConfig': health_check_config,
+        })
+        stubber.add_response('change_tags_for_resource', {})
+
+        id = provider.get_health_check_id(record, 'target-1.unit.tests.', True)
         self.assertEquals('42', id)
         stubber.assert_no_pending_responses()
 
@@ -946,6 +1174,8 @@ class TestRoute53Provider(TestCase):
             'CallerReference': ANY,
             'HealthCheckConfig': health_check_config,
         })
+        stubber.add_response('change_tags_for_resource', {})
+        stubber.add_response('change_tags_for_resource', {})
 
         record = Record.new(self.expected, 'a', {
             'ttl': 61,
@@ -965,7 +1195,8 @@ class TestRoute53Provider(TestCase):
             }
         })
 
-        id = provider.get_health_check_id(record, 'AF', record.geo['AF'], True)
+        value = record.geo['AF'].values[0]
+        id = provider.get_health_check_id(record, value, True)
         ml = provider.health_checks[id]['HealthCheckConfig']['MeasureLatency']
         self.assertEqual(False, ml)
 
@@ -1005,7 +1236,7 @@ class TestRoute53Provider(TestCase):
             'HealthCheckId': '44',
         })
         change = Create(record)
-        provider._mod_Create(change)
+        provider._mod_Create(change, 'z43')
         stubber.assert_no_pending_responses()
 
         # gc through _mod_Update
@@ -1014,7 +1245,7 @@ class TestRoute53Provider(TestCase):
         })
         # first record is ignored for our purposes, we have to pass something
         change = Update(record, record)
-        provider._mod_Create(change)
+        provider._mod_Create(change, 'z43')
         stubber.assert_no_pending_responses()
 
         # gc through _mod_Delete, expect 3 to go away, can't check order
@@ -1029,7 +1260,7 @@ class TestRoute53Provider(TestCase):
             'HealthCheckId': ANY,
         })
         change = Delete(record)
-        provider._mod_Delete(change)
+        provider._mod_Delete(change, 'z43')
         stubber.assert_no_pending_responses()
 
         # gc only AAAA, leave the A's alone
@@ -1385,6 +1616,119 @@ class TestRoute53Provider(TestCase):
         self.assertEquals(1, len(extra))
         stubber.assert_no_pending_responses()
 
+    def test_extra_change_dynamic_has_health_check(self):
+        provider, stubber = self._get_stubbed_provider()
+
+        list_hosted_zones_resp = {
+            'HostedZones': [{
+                'Name': 'unit.tests.',
+                'Id': 'z42',
+                'CallerReference': 'abc',
+            }],
+            'Marker': 'm',
+            'IsTruncated': False,
+            'MaxItems': '100',
+        }
+        stubber.add_response('list_hosted_zones', list_hosted_zones_resp, {})
+
+        # record with geo and no health check returns change
+        desired = Zone('unit.tests.', [])
+        record = Record.new(desired, 'a', {
+            'ttl': 30,
+            'type': 'A',
+            'value': '1.2.3.4',
+            'dynamic': {
+                'pools': {
+                    'one': {
+                        'values': [{
+                            'value': '2.2.3.4',
+                        }],
+                    },
+                },
+                'rules': [{
+                    'pool': 'one',
+                }],
+            },
+        })
+        desired.add_record(record)
+        list_resource_record_sets_resp = {
+            'ResourceRecordSets': [{
+                # other name
+                'Name': 'unit.tests.',
+                'Type': 'A',
+                'GeoLocation': {
+                    'CountryCode': '*',
+                },
+                'ResourceRecords': [{
+                    'Value': '1.2.3.4',
+                }],
+                'TTL': 61,
+            }, {
+                # matching name, other type
+                'Name': 'a.unit.tests.',
+                'Type': 'AAAA',
+                'ResourceRecords': [{
+                    'Value': '2001:0db8:3c4d:0015:0000:0000:1a2f:1a4b'
+                }],
+                'TTL': 61,
+            }, {
+                # default value pool
+                'Name': '_octodns-default-pool.a.unit.tests.',
+                'Type': 'A',
+                'GeoLocation': {
+                    'CountryCode': '*',
+                },
+                'ResourceRecords': [{
+                    'Value': '1.2.3.4',
+                }],
+                'TTL': 61,
+            }, {
+                # match
+                'Name': '_octodns-one-value.a.unit.tests.',
+                'Type': 'A',
+                'ResourceRecords': [{
+                    'Value': '2.2.3.4',
+                }],
+                'TTL': 61,
+                'HealthCheckId': '42',
+            }],
+            'IsTruncated': False,
+            'MaxItems': '100',
+        }
+        stubber.add_response('list_resource_record_sets',
+                             list_resource_record_sets_resp,
+                             {'HostedZoneId': 'z42'})
+        stubber.add_response('list_health_checks', {
+            'HealthChecks': [{
+                'Id': '42',
+                'CallerReference': self.caller_ref,
+                'HealthCheckConfig': {
+                    'Type': 'HTTPS',
+                    'FullyQualifiedDomainName': 'a.unit.tests',
+                    'IPAddress': '2.2.3.4',
+                    'ResourcePath': '/_dns',
+                    'Type': 'HTTPS',
+                    'Port': 443,
+                    'MeasureLatency': True
+                },
+                'HealthCheckVersion': 2,
+            }],
+            'IsTruncated': False,
+            'MaxItems': '100',
+            'Marker': '',
+        })
+        extra = provider._extra_changes(desired=desired, changes=[])
+        self.assertEquals(0, len(extra))
+        stubber.assert_no_pending_responses()
+
+        # change b/c of healthcheck path
+        record._octodns['healthcheck'] = {
+            'path': '/_ready'
+        }
+        extra = provider._extra_changes(desired=desired, changes=[])
+        self.assertEquals(1, len(extra))
+        stubber.assert_no_pending_responses()
+
         # change b/c of healthcheck host
         record._octodns['healthcheck'] = {
             'host': 'foo.bar.io'
@@ -1529,34 +1873,120 @@ class TestRoute53Provider(TestCase):
                           ._unique_id_handlers['retry-config-route53']
                           ['handler']._checker.__dict__['_max_attempts'])
 
+    def test_data_for_dynamic(self):
+        provider = Route53Provider('test', 'abc', '123')
+
+        data = provider._data_for_dynamic('', 'A', dynamic_rrsets)
+        self.assertEquals(dynamic_record_data, data)
+
+    @patch('octodns.provider.route53.Route53Provider._get_zone_id')
+    @patch('octodns.provider.route53.Route53Provider._load_records')
+    def test_dynamic_populate(self, load_records_mock, get_zone_id_mock):
+        provider = Route53Provider('test', 'abc', '123')
+
+        get_zone_id_mock.side_effect = ['z44']
+        load_records_mock.side_effect = [dynamic_rrsets]
+
+        got = Zone('unit.tests.', [])
+        provider.populate(got)
+
+        self.assertEquals(1, len(got.records))
+        record = list(got.records)[0]
+        self.assertEquals('', record.name)
+        self.assertEquals('A', record._type)
+        self.assertEquals([
+            '1.1.2.1',
+            '1.1.2.2',
+        ], record.values)
+        self.assertTrue(record.dynamic)
+
+        self.assertEquals({
+            'ap-southeast-1': {
+                'fallback': 'us-east-1',
+                'values': [{
+                    'weight': 2, 'value': '1.4.1.1'
+                }, {
+                    'weight': 2, 'value': '1.4.1.2'
+                }]
+            },
+            'eu-central-1': {
+                'fallback': 'us-east-1',
+                'values': [{
+                    'weight': 1, 'value': '1.3.1.1'
+                }, {
+                    'weight': 1, 'value': '1.3.1.2'
+                }],
+            },
+            'us-east-1': {
+                'fallback': None,
+                'values': [{
+                    'weight': 1, 'value': '1.5.1.1'
+                }, {
+                    'weight': 1, 'value': '1.5.1.2'
+                }],
+            }
+        }, {k: v.data for k, v in record.dynamic.pools.items()})
+
+        self.assertEquals([
+            {
+                'geos': ['AS-CN', 'AS-JP'],
+                'pool': 'ap-southeast-1',
+            }, {
+                'geos': ['EU', 'NA-US-FL'],
+                'pool': 'eu-central-1',
+            }, {
+                'pool': 'us-east-1',
+            }], [r.data for r in record.dynamic.rules])
+
 
 class TestRoute53Records(TestCase):
+    existing = Zone('unit.tests.', [])
+    record_a = Record.new(existing, '', {
+        'geo': {
+            'NA-US': ['2.2.2.2', '3.3.3.3'],
+            'OC': ['4.4.4.4', '5.5.5.5']
+        },
+        'ttl': 99,
+        'type': 'A',
+        'values': ['9.9.9.9']
+    })
+
+    def test_value_fors(self):
+        route53_record = _Route53Record(None, self.record_a, False)
+
+        for value in (None, '', 'foo', 'bar', '1.2.3.4'):
+            converted = route53_record._value_convert_value(value,
+                                                            self.record_a)
+            self.assertEquals(value, converted)
+
+        record_txt = Record.new(self.existing, 'txt', {
+            'ttl': 98,
+            'type': 'TXT',
+            'value': 'Not Important',
+        })
+
+        # We don't really have to test the details fo chunked_value as that's
+        # tested elsewhere, we just need to make sure that it's plumbed up and
+        # working
+        self.assertEquals('"Not Important"', route53_record
+                          ._value_convert_quoted(record_txt.values[0],
+                                                 record_txt))
 
     def test_route53_record(self):
-        existing = Zone('unit.tests.', [])
-        record_a = Record.new(existing, '', {
-            'geo': {
-                'NA-US': ['2.2.2.2', '3.3.3.3'],
-                'OC': ['4.4.4.4', '5.5.5.5']
-            },
-            'ttl': 99,
-            'type': 'A',
-            'values': ['9.9.9.9']
-        })
-        a = _Route53Record(None, record_a, False)
+        a = _Route53Record(None, self.record_a, False)
         self.assertEquals(a, a)
-        b = _Route53Record(None, Record.new(existing, '',
+        b = _Route53Record(None, Record.new(self.existing, '',
                                             {'ttl': 32, 'type': 'A',
                                              'values': ['8.8.8.8',
                                                         '1.1.1.1']}),
                            False)
         self.assertEquals(b, b)
-        c = _Route53Record(None, Record.new(existing, 'other',
+        c = _Route53Record(None, Record.new(self.existing, 'other',
                                             {'ttl': 99, 'type': 'A',
                                              'values': ['9.9.9.9']}),
                            False)
         self.assertEquals(c, c)
-        d = _Route53Record(None, Record.new(existing, '',
+        d = _Route53Record(None, Record.new(self.existing, '',
                                             {'ttl': 42, 'type': 'MX',
                                              'value': {
                                                  'preference': 10,
@@ -1572,7 +2002,7 @@ class TestRoute53Records(TestCase):
         self.assertNotEquals(a, c)
 
         # Same everything, different class is not the same
-        e = _Route53GeoDefault(None, record_a, False)
+        e = _Route53GeoDefault(None, self.record_a, False)
         self.assertNotEquals(a, e)
 
         class DummyProvider(object):
@@ -1581,11 +2011,11 @@ class TestRoute53Records(TestCase):
                 return None
 
         provider = DummyProvider()
-        f = _Route53GeoRecord(provider, record_a, 'NA-US',
-                              record_a.geo['NA-US'], False)
+        f = _Route53GeoRecord(provider, self.record_a, 'NA-US',
+                              self.record_a.geo['NA-US'], False)
         self.assertEquals(f, f)
-        g = _Route53GeoRecord(provider, record_a, 'OC',
-                              record_a.geo['OC'], False)
+        g = _Route53GeoRecord(provider, self.record_a, 'OC',
+                              self.record_a.geo['OC'], False)
         self.assertEquals(g, g)
 
         # Geo and non-geo are not the same, using Geo as primary to get it's
@@ -1598,3 +2028,303 @@ class TestRoute53Records(TestCase):
         a.__repr__()
         e.__repr__()
         f.__repr__()
+
+    def test_new_dynamic(self):
+        provider = Route53Provider('test', 'abc', '123')
+
+        # Just so boto won't try and make any calls
+        stubber = Stubber(provider._conn)
+        stubber.activate()
+
+        # We'll assume we create all healthchecks here, this functionality is
+        # thoroughly tested elsewhere
+        provider._health_checks = {}
+        # When asked for a healthcheck return dummy info
+        provider.get_health_check_id = lambda r, v, c: 'hc42'
+
+        zone = Zone('unit.tests.', [])
+        record = Record.new(zone, '', dynamic_record_data)
+
+        # Convert a record into _Route53Records
+        route53_records = _Route53Record.new(provider, record, 'z45',
+                                             creating=True)
+        self.assertEquals(18, len(route53_records))
+
+        # Convert the route53_records into mods
+        self.assertEquals([{
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'HealthCheckId': 'hc42',
+                'Name': '_octodns-ap-southeast-1-value.unit.tests.',
+                'ResourceRecords': [{
+                    'Value': '1.4.1.2'}],
+                'SetIdentifier': 'ap-southeast-1-001',
+                'TTL': 60,
+                'Type': 'A',
+                'Weight': 2
+            }
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'HealthCheckId': 'hc42',
+                'Name': '_octodns-ap-southeast-1-value.unit.tests.',
+                'ResourceRecords': [{
+                    'Value': '1.4.1.1'}],
+                'SetIdentifier': 'ap-southeast-1-000',
+                'TTL': 60,
+                'Type': 'A',
+                'Weight': 2
+            }
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'z45'
+                },
+                'GeoLocation': {
+                    'CountryCode': 'JP'},
+                'Name': 'unit.tests.',
+                'SetIdentifier': '0-ap-southeast-1-AS-JP',
+                'Type': 'A'
+            }
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': '_octodns-eu-central-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'z45'},
+                'GeoLocation': {
+                    'CountryCode': 'US',
+                    'SubdivisionCode': 'FL',
+                },
+                'Name': 'unit.tests.',
+                'SetIdentifier': '1-eu-central-1-NA-US-FL',
+                'Type': 'A'}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': '_octodns-us-east-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'z45'},
+                'GeoLocation': {
+                    'CountryCode': '*'},
+                'Name': 'unit.tests.',
+                'SetIdentifier': '2-us-east-1-None',
+                'Type': 'A'}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': '_octodns-us-east-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'z45'},
+                'Failover': 'SECONDARY',
+                'Name': '_octodns-ap-southeast-1-pool.unit.tests.',
+                'SetIdentifier': 'ap-southeast-1-Secondary-us-east-1',
+                'Type': 'A'}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': '_octodns-ap-southeast-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'z45'},
+                'GeoLocation': {
+                    'CountryCode': 'CN'},
+                'Name': 'unit.tests.',
+                'SetIdentifier': '0-ap-southeast-1-AS-CN',
+                'Type': 'A'}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': '_octodns-us-east-1-value.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'z45'},
+                'Failover': 'PRIMARY',
+                'Name': '_octodns-us-east-1-pool.unit.tests.',
+                'SetIdentifier': 'us-east-1-Primary',
+                'Type': 'A'}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': '_octodns-eu-central-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'z45'},
+                'GeoLocation': {
+                    'ContinentCode': 'EU'},
+                'Name': 'unit.tests.',
+                'SetIdentifier': '1-eu-central-1-EU',
+                'Type': 'A'}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': '_octodns-eu-central-1-value.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'z45'},
+                'Failover': 'PRIMARY',
+                'Name': '_octodns-eu-central-1-pool.unit.tests.',
+                'SetIdentifier': 'eu-central-1-Primary',
+                'Type': 'A'}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'Name': '_octodns-default-pool.unit.tests.',
+                'ResourceRecords': [{
+                    'Value': '1.1.2.1'},
+                    {
+                        'Value': '1.1.2.2'}],
+                'TTL': 60,
+                'Type': 'A'}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'HealthCheckId': 'hc42',
+                'Name': '_octodns-eu-central-1-value.unit.tests.',
+                'ResourceRecords': [{
+                    'Value': '1.3.1.2'}],
+                'SetIdentifier': 'eu-central-1-001',
+                'TTL': 60,
+                'Type': 'A',
+                'Weight': 1}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'HealthCheckId': 'hc42',
+                'Name': '_octodns-eu-central-1-value.unit.tests.',
+                'ResourceRecords': [{
+                    'Value': '1.3.1.1'}],
+                'SetIdentifier': 'eu-central-1-000',
+                'TTL': 60,
+                'Type': 'A',
+                'Weight': 1}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': '_octodns-default-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'z45'},
+                'Failover': 'SECONDARY',
+                'Name': '_octodns-us-east-1-pool.unit.tests.',
+                'SetIdentifier': 'us-east-1-Secondary-default',
+                'Type': 'A'}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'HealthCheckId': 'hc42',
+                'Name': '_octodns-us-east-1-value.unit.tests.',
+                'ResourceRecords': [{
+                    'Value': '1.5.1.2'}],
+                'SetIdentifier': 'us-east-1-001',
+                'TTL': 60,
+                'Type': 'A',
+                'Weight': 1}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'HealthCheckId': 'hc42',
+                'Name': '_octodns-us-east-1-value.unit.tests.',
+                'ResourceRecords': [{
+                    'Value': '1.5.1.1'}],
+                'SetIdentifier': 'us-east-1-000',
+                'TTL': 60,
+                'Type': 'A',
+                'Weight': 1}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': '_octodns-ap-southeast-1-value.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'z45'},
+                'Failover': 'PRIMARY',
+                'Name': '_octodns-ap-southeast-1-pool.unit.tests.',
+                'SetIdentifier': 'ap-southeast-1-Primary',
+                'Type': 'A'}
+        }, {
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'AliasTarget': {
+                    'DNSName': '_octodns-us-east-1-pool.unit.tests.',
+                    'EvaluateTargetHealth': True,
+                    'HostedZoneId': 'z45'},
+                'Failover': 'SECONDARY',
+                'Name': '_octodns-eu-central-1-pool.unit.tests.',
+                'SetIdentifier': 'eu-central-1-Secondary-us-east-1',
+                'Type': 'A'}
+        }], [r.mod('CREATE') for r in route53_records])
+
+        for route53_record in route53_records:
+            # Smoke test stringification
+            route53_record.__repr__()
+
+
+class TestModKeyer(TestCase):
+
+    def test_mod_keyer(self):
+
+        # First "column"
+
+        # Deletes come first
+        self.assertEquals((0, 0, 'something'), _mod_keyer({
+            'Action': 'DELETE',
+            'ResourceRecordSet': {
+                'Name': 'something',
+            }
+        }))
+
+        # Creates come next
+        self.assertEquals((1, 0, 'another'), _mod_keyer({
+            'Action': 'CREATE',
+            'ResourceRecordSet': {
+                'Name': 'another',
+            }
+        }))
+
+        # Then upserts
+        self.assertEquals((2, 0, 'last'), _mod_keyer({
+            'Action': 'UPSERT',
+            'ResourceRecordSet': {
+                'Name': 'last',
+            }
+        }))
+
+        # Second "column" value records tested above
+
+        # AliasTarget primary second (to value)
+        self.assertEquals((0, 1, 'thing'), _mod_keyer({
+            'Action': 'DELETE',
+            'ResourceRecordSet': {
+                'AliasTarget': 'some-target',
+                'Failover': 'PRIMARY',
+                'Name': 'thing',
+            }
+        }))
+
+        # AliasTarget secondary third
+        self.assertEquals((0, 2, 'thing'), _mod_keyer({
+            'Action': 'DELETE',
+            'ResourceRecordSet': {
+                'AliasTarget': 'some-target',
+                'Failover': 'SECONDARY',
+                'Name': 'thing',
+            }
+        }))
+
+        # GeoLocation fourth
+        self.assertEquals((0, 3, 'some-id'), _mod_keyer({
+            'Action': 'DELETE',
+            'ResourceRecordSet': {
+                'GeoLocation': 'some-target',
+                'SetIdentifier': 'some-id',
+            }
+        }))
+
+        # The third "column" has already been tested above, Name/SetIdentifier