Merge pull request #165 from procore/master

Add RackSpace as a provider

README.md

@@ -158,6 +158,7 @@ The above command pulled the existing data out of Route53 and placed the results
 | [Ns1Provider](/octodns/provider/ns1.py) | All | No | |
 | [OVH](/octodns/provider/ovh.py) | A, AAAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, SSHFP, TXT | No | |
 | [PowerDnsProvider](/octodns/provider/powerdns.py) | All | No | |
+| [Rackspace](/octodns/provider/rackspace.py) | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No |  |
 | [Route53](/octodns/provider/route53.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Yes | |
 | [TinyDNSSource](/octodns/source/tinydns.py) | A, CNAME, MX, NS, PTR | No | read-only |
 | [YamlProvider](/octodns/provider/yaml.py) | All | Yes | config |

octodns/provider/rackspace.py

@@ -0,0 +1,383 @@
+#
+#
+#
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from requests import HTTPError, Session, post
+from collections import defaultdict
+import logging
+import string
+import time
+
+from ..record import Record
+from .base import BaseProvider
+
+
+def add_trailing_dot(s):
+    assert s
+    assert s[-1] != '.'
+    return s + '.'
+
+
+def remove_trailing_dot(s):
+    assert s
+    assert s[-1] == '.'
+    return s[:-1]
+
+
+def escape_semicolon(s):
+    assert s
+    return string.replace(s, ';', '\;')
+
+
+def unescape_semicolon(s):
+    assert s
+    return string.replace(s, '\;', ';')
+
+
+class RackspaceProvider(BaseProvider):
+    SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NS', 'PTR', 'SPF',
+                    'TXT'))
+    TIMEOUT = 5
+
+    def __init__(self, id, username, api_key, ratelimit_delay=0.0, *args,
+                 **kwargs):
+        '''
+        Rackspace API v1 Provider
+
+        rackspace:
+            class: octodns.provider.rackspace.RackspaceProvider
+            # The the username to authenticate with (required)
+            username: username
+            # The api key that grants access for that user (required)
+            api_key: api-key
+        '''
+        self.log = logging.getLogger('RackspaceProvider[{}]'.format(id))
+        super(RackspaceProvider, self).__init__(id, *args, **kwargs)
+
+        auth_token, dns_endpoint = self._get_auth_token(username, api_key)
+        self.dns_endpoint = dns_endpoint
+
+        self.ratelimit_delay = float(ratelimit_delay)
+
+        sess = Session()
+        sess.headers.update({'X-Auth-Token': auth_token})
+        self._sess = sess
+
+        # Map record type, name, and data to an id when populating so that
+        # we can find the id for update and delete operations.
+        self._id_map = {}
+
+    def _get_auth_token(self, username, api_key):
+        ret = post('https://identity.api.rackspacecloud.com/v2.0/tokens',
+                   json={"auth": {
+                       "RAX-KSKEY:apiKeyCredentials": {"username": username,
+                                                       "apiKey": api_key}}},
+                   )
+        cloud_dns_endpoint = \
+            [x for x in ret.json()['access']['serviceCatalog'] if
+             x['name'] == 'cloudDNS'][0]['endpoints'][0]['publicURL']
+        return ret.json()['access']['token']['id'], cloud_dns_endpoint
+
+    def _get_zone_id_for(self, zone):
+        ret = self._request('GET', 'domains', pagination_key='domains')
+        return [x for x in ret if x['name'] == zone.name[:-1]][0]['id']
+
+    def _request(self, method, path, data=None, pagination_key=None):
+        self.log.debug('_request: method=%s, path=%s', method, path)
+        url = '{}/{}'.format(self.dns_endpoint, path)
+
+        if pagination_key:
+            resp = self._paginated_request_for_url(method, url, data,
+                                                   pagination_key)
+        else:
+            resp = self._request_for_url(method, url, data)
+        time.sleep(self.ratelimit_delay)
+        return resp
+
+    def _request_for_url(self, method, url, data):
+        resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT)
+        self.log.debug('_request:   status=%d', resp.status_code)
+        resp.raise_for_status()
+        return resp
+
+    def _paginated_request_for_url(self, method, url, data, pagination_key):
+        acc = []
+
+        resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT)
+        self.log.debug('_request:   status=%d', resp.status_code)
+        resp.raise_for_status()
+        acc.extend(resp.json()[pagination_key])
+
+        next_page = [x for x in resp.json().get('links', []) if
+                     x['rel'] == 'next']
+        if next_page:
+            url = next_page[0]['href']
+            acc.extend(self._paginated_request_for_url(method, url, data,
+                                                       pagination_key))
+            return acc
+        else:
+            return acc
+
+    def _post(self, path, data=None):
+        return self._request('POST', path, data=data)
+
+    def _put(self, path, data=None):
+        return self._request('PUT', path, data=data)
+
+    def _delete(self, path, data=None):
+        return self._request('DELETE', path, data=data)
+
+    @staticmethod
+    def _as_unicode(s, codec):
+        if not isinstance(s, unicode):
+            return unicode(s, codec)
+        return s
+
+    @classmethod
+    def _key_for_record(cls, rs_record):
+        return cls._as_unicode(rs_record['type'], 'ascii'), \
+            cls._as_unicode(rs_record['name'], 'utf-8'), \
+            cls._as_unicode(rs_record['data'], 'utf-8')
+
+    def _data_for_multiple(self, rrset):
+        return {
+            'type': rrset[0]['type'],
+            'values': [r['data'] for r in rrset],
+            'ttl': rrset[0]['ttl']
+        }
+
+    _data_for_A = _data_for_multiple
+    _data_for_AAAA = _data_for_multiple
+
+    def _data_for_NS(self, rrset):
+        return {
+            'type': rrset[0]['type'],
+            'values': [add_trailing_dot(r['data']) for r in rrset],
+            'ttl': rrset[0]['ttl']
+        }
+
+    def _data_for_single(self, record):
+        return {
+            'type': record[0]['type'],
+            'value': add_trailing_dot(record[0]['data']),
+            'ttl': record[0]['ttl']
+        }
+
+    _data_for_ALIAS = _data_for_single
+    _data_for_CNAME = _data_for_single
+    _data_for_PTR = _data_for_single
+
+    def _data_for_textual(self, rrset):
+        return {
+            'type': rrset[0]['type'],
+            'values': [escape_semicolon(r['data']) for r in rrset],
+            'ttl': rrset[0]['ttl']
+        }
+
+    _data_for_SPF = _data_for_textual
+    _data_for_TXT = _data_for_textual
+
+    def _data_for_MX(self, rrset):
+        values = []
+        for record in rrset:
+            values.append({
+                'priority': record['priority'],
+                'value': add_trailing_dot(record['data']),
+            })
+        return {
+            'type': rrset[0]['type'],
+            'values': values,
+            'ttl': rrset[0]['ttl']
+        }
+
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s', zone.name)
+        resp_data = None
+        try:
+            domain_id = self._get_zone_id_for(zone)
+            resp_data = self._request('GET',
+                                      'domains/{}/records'.format(domain_id),
+                                      pagination_key='records')
+            self.log.debug('populate:   loaded')
+        except HTTPError as e:
+            if e.response.status_code == 401:
+                # Nicer error message for auth problems
+                raise Exception('Rackspace request unauthorized')
+            elif e.response.status_code == 404:
+                # Zone not found leaves the zone empty instead of failing.
+                return
+            raise
+
+        before = len(zone.records)
+
+        if resp_data:
+            records = self._group_records(resp_data)
+            for record_type, records_of_type in records.items():
+                for raw_record_name, record_set in records_of_type.items():
+                    data_for = getattr(self,
+                                       '_data_for_{}'.format(record_type))
+                    record_name = zone.hostname_from_fqdn(raw_record_name)
+                    record = Record.new(zone, record_name,
+                                        data_for(record_set),
+                                        source=self)
+                    zone.add_record(record)
+
+        self.log.info('populate:   found %s records',
+                      len(zone.records) - before)
+
+    def _group_records(self, all_records):
+        records = defaultdict(lambda: defaultdict(list))
+        for record in all_records:
+            self._id_map[self._key_for_record(record)] = record['id']
+            records[record['type']][record['name']].append(record)
+        return records
+
+    @staticmethod
+    def _record_for_single(record, value):
+        return {
+            'name': remove_trailing_dot(record.fqdn),
+            'type': record._type,
+            'data': value,
+            'ttl': max(record.ttl, 300),
+        }
+
+    _record_for_A = _record_for_single
+    _record_for_AAAA = _record_for_single
+
+    @staticmethod
+    def _record_for_named(record, value):
+        return {
+            'name': remove_trailing_dot(record.fqdn),
+            'type': record._type,
+            'data': remove_trailing_dot(value),
+            'ttl': max(record.ttl, 300),
+        }
+
+    _record_for_NS = _record_for_named
+    _record_for_ALIAS = _record_for_named
+    _record_for_CNAME = _record_for_named
+    _record_for_PTR = _record_for_named
+
+    @staticmethod
+    def _record_for_textual(record, value):
+        return {
+            'name': remove_trailing_dot(record.fqdn),
+            'type': record._type,
+            'data': unescape_semicolon(value),
+            'ttl': max(record.ttl, 300),
+        }
+
+    _record_for_SPF = _record_for_textual
+    _record_for_TXT = _record_for_textual
+
+    @staticmethod
+    def _record_for_MX(record, value):
+        return {
+            'name': remove_trailing_dot(record.fqdn),
+            'type': record._type,
+            'data': remove_trailing_dot(value.exchange),
+            'ttl': max(record.ttl, 300),
+            'priority': value.preference
+        }
+
+    def _get_values(self, record):
+        try:
+            return record.values
+        except AttributeError:
+            return [record.value]
+
+    def _mod_Create(self, change):
+        return self._create_given_change_values(change,
+                                                self._get_values(change.new))
+
+    def _create_given_change_values(self, change, values):
+        transformer = getattr(self, "_record_for_{}".format(change.new._type))
+        return [transformer(change.new, v) for v in values]
+
+    def _mod_Update(self, change):
+        existing_values = self._get_values(change.existing)
+        new_values = self._get_values(change.new)
+
+        # A reduction in number of values in an update record needs
+        # to get upgraded into a Delete change for the removed values.
+        deleted_values = set(existing_values) - set(new_values)
+        delete_out = self._delete_given_change_values(change, deleted_values)
+
+        # An increase in number of values in an update record needs
+        # to get upgraded into a Create change for the added values.
+        create_values = set(new_values) - set(existing_values)
+        create_out = self._create_given_change_values(change, create_values)
+
+        update_out = []
+        update_values = set(new_values).intersection(set(existing_values))
+        for value in update_values:
+            transformer = getattr(self,
+                                  "_record_for_{}".format(change.new._type))
+            prior_rs_record = transformer(change.existing, value)
+            prior_key = self._key_for_record(prior_rs_record)
+            next_rs_record = transformer(change.new, value)
+            next_key = self._key_for_record(next_rs_record)
+            next_rs_record["id"] = self._id_map[prior_key]
+            del next_rs_record["type"]
+            update_out.append(next_rs_record)
+            self._id_map[next_key] = self._id_map[prior_key]
+            del self._id_map[prior_key]
+        return create_out, update_out, delete_out
+
+    def _mod_Delete(self, change):
+        return self._delete_given_change_values(change, self._get_values(
+            change.existing))
+
+    def _delete_given_change_values(self, change, values):
+        transformer = getattr(self, "_record_for_{}".format(
+            change.existing._type))
+        out = []
+        for value in values:
+            rs_record = transformer(change.existing, value)
+            key = self._key_for_record(rs_record)
+            out.append('id=' + self._id_map[key])
+            del self._id_map[key]
+        return out
+
+    def _apply(self, plan):
+        desired = plan.desired
+        changes = plan.changes
+        self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name,
+                       len(changes))
+
+        # Creates, updates, and deletes are processed by different endpoints
+        # and are broken out by record-set entries; pre-process everything
+        # into these buckets in order to minimize the number of API calls.
+        domain_id = self._get_zone_id_for(desired)
+        creates = []
+        updates = []
+        deletes = []
+        for change in changes:
+            if change.__class__.__name__ == 'Create':
+                creates += self._mod_Create(change)
+            elif change.__class__.__name__ == 'Update':
+                add_creates, add_updates, add_deletes = self._mod_Update(
+                    change)
+                creates += add_creates
+                updates += add_updates
+                deletes += add_deletes
+            else:
+                assert change.__class__.__name__ == 'Delete'
+                deletes += self._mod_Delete(change)
+
+        if deletes:
+            params = "&".join(sorted(deletes))
+            self._delete('domains/{}/records?{}'.format(domain_id, params))
+
+        if updates:
+            data = {"records": sorted(updates, key=lambda v: v['name'])}
+            self._put('domains/{}/records'.format(domain_id), data=data)
+
+        if creates:
+            data = {"records": sorted(creates, key=lambda v: v['type'] +
+                                      v['name'] +
+                                      v.get('data', ''))}
+            self._post('domains/{}/records'.format(domain_id), data=data)

tests/fixtures/rackspace-auth-response.json

@@ -0,0 +1,87 @@
+{
+        "access": {
+                "token": {
+                        "id": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+                        "expires": "2014-11-24T22:05:39.115Z",
+                        "tenant": {
+                                "id": "110011",
+                                "name": "110011"
+                        },
+                        "RAX-AUTH:authenticatedBy": [
+                                "APIKEY"
+                        ]
+                },
+                "serviceCatalog": [
+                        {
+                                "name": "cloudDatabases",
+                                "endpoints": [
+                                        {
+                                                "publicURL": "https://syd.databases.api.rackspacecloud.com/v1.0/110011",
+                                                "region": "SYD",
+                                                "tenantId": "110011"
+                                        },
+                                        {
+                                                "publicURL": "https://dfw.databases.api.rackspacecloud.com/v1.0/110011",
+                                                "region": "DFW",
+                                                "tenantId": "110011"
+                                        },
+                                        {
+                                                "publicURL": "https://ord.databases.api.rackspacecloud.com/v1.0/110011",
+                                                "region": "ORD",
+                                                "tenantId": "110011"
+                                        },
+                                        {
+                                                "publicURL": "https://iad.databases.api.rackspacecloud.com/v1.0/110011",
+                                                "region": "IAD",
+                                                "tenantId": "110011"
+                                        },
+                                        {
+                                                "publicURL": "https://hkg.databases.api.rackspacecloud.com/v1.0/110011",
+                                                "region": "HKG",
+                                                "tenantId": "110011"
+                                        }
+                                ],
+                                "type": "rax:database"
+                        },
+                        {
+                                "name": "cloudDNS",
+                                "endpoints": [
+                                        {
+                                                "publicURL": "https://dns.api.rackspacecloud.com/v1.0/110011",
+                                                "tenantId": "110011"
+                                        }
+                                ],
+                                "type": "rax:dns"
+                        },
+                        {
+                                "name": "rackCDN",
+                                "endpoints": [
+                                        {
+                                                "internalURL": "https://global.cdn.api.rackspacecloud.com/v1.0/110011",
+                                                "publicURL": "https://global.cdn.api.rackspacecloud.com/v1.0/110011",
+                                                "tenantId": "110011"
+                                        }
+                                ],
+                                "type": "rax:cdn"
+                        }
+                ],
+                "user": {
+                        "id": "123456",
+                        "roles": [
+                                {
+                                        "description": "A Role that allows a user access to keystone Service methods",
+                                        "id": "6",
+                                        "name": "compute:default",
+                                        "tenantId": "110011"
+                                },
+                                {
+                                        "description": "User Admin Role.",
+                                        "id": "3",
+                                        "name": "identity:user-admin"
+                                }
+                        ],
+                        "name": "jsmith",
+                        "RAX-AUTH:defaultRegion": "ORD"
+                }
+        }
+}

tests/fixtures/rackspace-list-domains-response.json

@@ -0,0 +1,68 @@
+{
+  "totalEntries" : 10,
+  "domains" : [ {
+    "name" : "example.com",
+    "id" : 2725233,
+    "comment" : "Optional domain comment...",
+    "updated" : "2011-06-24T01:23:15.000+0000",
+    "accountId" : 1234,
+    "emailAddress" : "[email protected]",
+    "created" : "2011-06-24T01:12:51.000+0000"
+  }, {
+    "name" : "sub1.example.com",
+    "id" : 2725257,
+    "comment" : "1st sample subdomain",
+    "updated" : "2011-06-23T03:09:34.000+0000",
+    "accountId" : 1234,
+    "emailAddress" : "[email protected]",
+    "created" : "2011-06-23T03:09:33.000+0000"
+  }, {
+    "name" : "sub2.example.com",
+    "id" : 2725258,
+    "comment" : "1st sample subdomain",
+    "updated" : "2011-06-23T03:52:55.000+0000",
+    "accountId" : 1234,
+    "emailAddress" : "[email protected]",
+    "created" : "2011-06-23T03:52:55.000+0000"
+  }, {
+    "name" : "north.example.com",
+    "id" : 2725260,
+    "updated" : "2011-06-23T03:53:10.000+0000",
+    "accountId" : 1234,
+    "emailAddress" : "[email protected]",
+    "created" : "2011-06-23T03:53:09.000+0000"
+  }, {
+    "name" : "south.example.com",
+    "id" : 2725261,
+    "comment" : "Final sample subdomain",
+    "updated" : "2011-06-23T03:53:14.000+0000",
+    "accountId" : 1234,
+    "emailAddress" : "[email protected]",
+    "created" : "2011-06-23T03:53:14.000+0000"
+  }, {
+    "name" : "region2.example.net",
+    "id" : 2725352,
+    "updated" : "2011-06-23T20:21:06.000+0000",
+    "accountId" : 1234,
+    "created" : "2011-06-23T19:24:27.000+0000"
+  }, {
+    "name" : "example.org",
+    "id" : 2718984,
+    "updated" : "2011-05-03T14:47:32.000+0000",
+    "accountId" : 1234,
+    "created" : "2011-05-03T14:47:30.000+0000"
+  }, {
+    "name" : "rackspace.example",
+    "id" : 2722346,
+    "updated" : "2011-06-21T15:54:31.000+0000",
+    "accountId" : 1234,
+    "created" : "2011-06-15T19:02:07.000+0000"
+  }, {
+    "name" : "unit.tests",
+    "id" : 2722347,
+    "comment" : "Sample comment",
+    "updated" : "2011-06-21T15:54:31.000+0000",
+    "accountId" : 1234,
+    "created" : "2011-06-15T19:02:07.000+0000"
+  } ]
+}

tests/fixtures/rackspace-sample-recordset-existing-nameservers.json

@@ -0,0 +1,29 @@
+{
+  "totalEntries" : 3,
+  "records" : [{
+    "name" : "unit.tests.",
+    "id" : "A-6822995",
+    "type" : "A",
+    "data" : "1.2.3.4",
+    "updated" : "2011-06-24T01:12:53.000+0000",
+    "ttl" : 600,
+    "created" : "2011-06-24T01:12:53.000+0000"
+  }, {
+    "name" : "unit.tests.",
+    "id" : "NS-454454",
+    "type" : "NS",
+    "data" : "ns1.example.com",
+    "updated" : "2011-06-24T01:12:51.000+0000",
+    "ttl" : 600,
+    "created" : "2011-06-24T01:12:51.000+0000"
+  }, {
+    "name" : "unit.tests.",
+    "id" : "NS-454455",
+    "type" : "NS",
+    "data" : "ns2.example.com",
+    "updated" : "2011-06-24T01:12:52.000+0000",
+    "ttl" : 600,
+    "created" : "2011-06-24T01:12:52.000+0000"
+  }],
+  "links" : []
+}

tests/fixtures/rackspace-sample-recordset-page1.json

@@ -0,0 +1,33 @@
+{
+  "totalEntries" : 6,
+  "records" : [ {
+    "name" : "ftp.example.com",
+    "id" : "A-6817754",
+    "type" : "A",
+    "data" : "192.0.2.8",
+    "updated" : "2011-05-19T13:07:08.000+0000",
+    "ttl" : 5771,
+    "created" : "2011-05-18T19:53:09.000+0000"
+  }, {
+    "name" : "example.com",
+    "id" : "A-6822994",
+    "type" : "A",
+    "data" : "192.0.2.17",
+    "updated" : "2011-06-24T01:12:52.000+0000",
+    "ttl" : 86400,
+    "created" : "2011-06-24T01:12:52.000+0000"
+  }, {
+    "name" : "example.com",
+    "id" : "NS-6251982",
+    "type" : "NS",
+    "data" : "ns.rackspace.com",
+    "updated" : "2011-06-24T01:12:51.000+0000",
+    "ttl" : 3600,
+    "created" : "2011-06-24T01:12:51.000+0000"
+  } ],
+  "links" : [ {
+    "content" : "",
+    "href" : "https://localhost/v1.0/1234/domains/domain_id/records?limit=3&offset=3",
+    "rel" : "next"
+  } ]
+}

tests/fixtures/rackspace-sample-recordset-page2.json

@@ -0,0 +1,35 @@
+{
+  "totalEntries" : 6,
+  "records" : [ {
+    "name" : "example.com",
+    "id" : "NS-6251983",
+    "type" : "NS",
+    "data" : "ns2.rackspace.com",
+    "updated" : "2011-06-24T01:12:51.000+0000",
+    "ttl" : 3600,
+    "created" : "2011-06-24T01:12:51.000+0000"
+  }, {
+    "name" : "example.com",
+    "priority" : 5,
+    "id" : "MX-3151218",
+    "type" : "MX",
+    "data" : "mail.example.com",
+    "updated" : "2011-06-24T01:12:53.000+0000",
+    "ttl" : 3600,
+    "created" : "2011-06-24T01:12:53.000+0000"
+  }, {
+    "name" : "www.example.com",
+    "id" : "CNAME-9778009",
+    "type" : "CNAME",
+    "comment" : "This is a comment on the CNAME record",
+    "data" : "example.com",
+    "updated" : "2011-06-24T01:12:54.000+0000",
+    "ttl" : 5400,
+    "created" : "2011-06-24T01:12:54.000+0000"
+  } ],
+  "links" : [ {
+    "content" : "",
+    "href" : "https://dns.api.rackspacecloud.com/v1.0/1234/domains/domain_id/records?limit=3&offset=0",
+    "rel" : "previous"
+  }]
+}

tests/test_octodns_provider_rackspace.py

@@ -0,0 +1,864 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+import json
+import re
+from unittest import TestCase
+from urlparse import urlparse
+
+from requests import HTTPError
+from requests_mock import ANY, mock as requests_mock
+
+from octodns.provider.rackspace import RackspaceProvider
+from octodns.record import Record
+from octodns.zone import Zone
+
+EMPTY_TEXT = '''
+{
+  "totalEntries" : 0,
+  "records" : []
+}
+'''
+
+with open('./tests/fixtures/rackspace-auth-response.json') as fh:
+    AUTH_RESPONSE = fh.read()
+
+with open('./tests/fixtures/rackspace-list-domains-response.json') as fh:
+    LIST_DOMAINS_RESPONSE = fh.read()
+
+with open('./tests/fixtures/rackspace-sample-recordset-page1.json') as fh:
+    RECORDS_PAGE_1 = fh.read()
+
+with open('./tests/fixtures/rackspace-sample-recordset-page2.json') as fh:
+    RECORDS_PAGE_2 = fh.read()
+
+
+class TestRackspaceProvider(TestCase):
+    def setUp(self):
+        self.maxDiff = 1000
+        with requests_mock() as mock:
+            mock.post(ANY, status_code=200, text=AUTH_RESPONSE)
+            self.provider = RackspaceProvider('identity', 'test', 'api-key',
+                                              '0')
+            self.assertTrue(mock.called_once)
+
+    def test_bad_auth(self):
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=401, text='Unauthorized')
+
+            with self.assertRaises(Exception) as ctx:
+                zone = Zone('unit.tests.', [])
+                self.provider.populate(zone)
+            self.assertTrue('unauthorized' in ctx.exception.message)
+            self.assertTrue(mock.called_once)
+
+    def test_server_error(self):
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=502, text='Things caught fire')
+
+            with self.assertRaises(HTTPError) as ctx:
+                zone = Zone('unit.tests.', [])
+                self.provider.populate(zone)
+            self.assertEquals(502, ctx.exception.response.status_code)
+            self.assertTrue(mock.called_once)
+
+    def test_nonexistent_zone(self):
+        # Non-existent zone doesn't populate anything
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=404,
+                     json={'error': "Could not find domain 'unit.tests.'"})
+
+            zone = Zone('unit.tests.', [])
+            self.provider.populate(zone)
+            self.assertEquals(set(), zone.records)
+            self.assertTrue(mock.called_once)
+
+    def test_multipage_populate(self):
+        with requests_mock() as mock:
+            mock.get(re.compile('domains$'), status_code=200,
+                     text=LIST_DOMAINS_RESPONSE)
+            mock.get(re.compile('records'), status_code=200,
+                     text=RECORDS_PAGE_1)
+            mock.get(re.compile('records.*offset=3'), status_code=200,
+                     text=RECORDS_PAGE_2)
+
+            zone = Zone('unit.tests.', [])
+            self.provider.populate(zone)
+            self.assertEquals(5, len(zone.records))
+
+    def test_plan_disappearing_ns_records(self):
+        expected = Zone('unit.tests.', [])
+        expected.add_record(Record.new(expected, '', {
+            'type': 'NS',
+            'ttl': 600,
+            'values': ['8.8.8.8.', '9.9.9.9.']
+        }))
+        expected.add_record(Record.new(expected, 'sub', {
+            'type': 'NS',
+            'ttl': 600,
+            'values': ['8.8.8.8.', '9.9.9.9.']
+        }))
+        with requests_mock() as mock:
+            mock.get(re.compile('domains$'), status_code=200,
+                     text=LIST_DOMAINS_RESPONSE)
+            mock.get(re.compile('records'), status_code=200, text=EMPTY_TEXT)
+
+            plan = self.provider.plan(expected)
+            self.assertTrue(mock.called)
+
+            # OctoDNS does not propagate top-level NS records.
+            self.assertEquals(1, len(plan.changes))
+
+    def test_fqdn_a_record(self):
+        expected = Zone('example.com.', [])
+        # expected.add_record(Record.new(expected, 'foo', '1.2.3.4'))
+
+        with requests_mock() as list_mock:
+            list_mock.get(re.compile('domains$'), status_code=200,
+                          text=LIST_DOMAINS_RESPONSE)
+            list_mock.get(re.compile('records'), status_code=200,
+                          json={'records': [
+                              {'type': 'A',
+                               'name': 'foo.example.com',
+                               'id': 'A-111111',
+                               'data': '1.2.3.4',
+                               'ttl': 300}]})
+            plan = self.provider.plan(expected)
+            self.assertTrue(list_mock.called)
+            self.assertEqual(1, len(plan.changes))
+            self.assertTrue(
+                plan.changes[0].existing.fqdn == 'foo.example.com.')
+
+        with requests_mock() as mock:
+            def _assert_deleting(request, context):
+                parts = urlparse(request.url)
+                self.assertEqual('id=A-111111', parts.query)
+
+            mock.get(re.compile('domains$'), status_code=200,
+                     text=LIST_DOMAINS_RESPONSE)
+            mock.delete(re.compile('domains/.*/records?.*'), status_code=202,
+                        text=_assert_deleting)
+            self.provider.apply(plan)
+            self.assertTrue(mock.called)
+
+    def _test_apply_with_data(self, data):
+        expected = Zone('unit.tests.', [])
+        for record in data.OtherRecords:
+            expected.add_record(
+                Record.new(expected, record['subdomain'], record['data']))
+
+        with requests_mock() as list_mock:
+            list_mock.get(re.compile('domains$'), status_code=200,
+                          text=LIST_DOMAINS_RESPONSE)
+            list_mock.get(re.compile('records'), status_code=200,
+                          json=data.OwnRecords)
+            plan = self.provider.plan(expected)
+            self.assertTrue(list_mock.called)
+            if not data.ExpectChanges:
+                self.assertFalse(plan)
+                return
+
+        with requests_mock() as mock:
+            called = set()
+
+            def make_assert_sending_right_body(expected):
+                def _assert_sending_right_body(request, _context):
+                    called.add(request.method)
+                    if request.method != 'DELETE':
+                        self.assertEqual(request.headers['content-type'],
+                                         'application/json')
+                        self.assertDictEqual(expected,
+                                             json.loads(request.body))
+                    else:
+                        parts = urlparse(request.url)
+                        self.assertEqual(expected, parts.query)
+                    return ''
+
+                return _assert_sending_right_body
+
+            mock.get(re.compile('domains$'), status_code=200,
+                     text=LIST_DOMAINS_RESPONSE)
+            mock.post(re.compile('domains/.*/records$'), status_code=202,
+                      text=make_assert_sending_right_body(
+                          data.ExpectedAdditions))
+            mock.delete(re.compile('domains/.*/records?.*'), status_code=202,
+                        text=make_assert_sending_right_body(
+                            data.ExpectedDeletions))
+            mock.put(re.compile('domains/.*/records$'), status_code=202,
+                     text=make_assert_sending_right_body(data.ExpectedUpdates))
+
+            self.provider.apply(plan)
+            self.assertTrue(data.ExpectedAdditions is None or "POST" in called)
+            self.assertTrue(
+                data.ExpectedDeletions is None or "DELETE" in called)
+            self.assertTrue(data.ExpectedUpdates is None or "PUT" in called)
+
+    def test_apply_no_change_empty(self):
+        class TestData(object):
+            OtherRecords = []
+            OwnRecords = {
+                "totalEntries": 0,
+                "records": []
+            }
+            ExpectChanges = False
+            ExpectedAdditions = None
+            ExpectedDeletions = None
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_no_change_a_records(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": '',
+                    "data": {
+                        'type': 'A',
+                        'ttl': 300,
+                        'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6']
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 3,
+                "records": [{
+                    "name": "unit.tests",
+                    "id": "A-111111",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }, {
+                    "name": "unit.tests",
+                    "id": "A-222222",
+                    "type": "A",
+                    "data": "1.2.3.5",
+                    "ttl": 300
+                }, {
+                    "name": "unit.tests",
+                    "id": "A-333333",
+                    "type": "A",
+                    "data": "1.2.3.6",
+                    "ttl": 300
+                }]
+            }
+            ExpectChanges = False
+            ExpectedAdditions = None
+            ExpectedDeletions = None
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_no_change_a_records_cross_zone(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": 'foo',
+                    "data": {
+                        'type': 'A',
+                        'ttl': 300,
+                        'value': '1.2.3.4'
+                    }
+                },
+                {
+                    "subdomain": 'bar',
+                    "data": {
+                        'type': 'A',
+                        'ttl': 300,
+                        'value': '1.2.3.4'
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 3,
+                "records": [{
+                    "name": "foo.unit.tests",
+                    "id": "A-111111",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }, {
+                    "name": "bar.unit.tests",
+                    "id": "A-222222",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }]
+            }
+            ExpectChanges = False
+            ExpectedAdditions = None
+            ExpectedDeletions = None
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_one_addition(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": '',
+                    "data": {
+                        'type': 'A',
+                        'ttl': 300,
+                        'value': '1.2.3.4'
+                    }
+                },
+                {
+                    "subdomain": 'foo',
+                    "data": {
+                        'type': 'NS',
+                        'ttl': 300,
+                        'value': 'ns.example.com.'
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 0,
+                "records": []
+            }
+            ExpectChanges = True
+            ExpectedAdditions = {
+                "records": [{
+                    "name": "unit.tests",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }, {
+                    "name": "foo.unit.tests",
+                    "type": "NS",
+                    "data": "ns.example.com",
+                    "ttl": 300
+                }]
+            }
+            ExpectedDeletions = None
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_create_MX(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": '',
+                    "data": {
+                        'type': 'MX',
+                        'ttl': 300,
+                        'value': {
+                            'value': 'mail1.example.com.',
+                            'priority': 1,
+                        }
+                    }
+                },
+                {
+                    "subdomain": 'foo',
+                    "data": {
+                        'type': 'MX',
+                        'ttl': 300,
+                        'value': {
+                            'value': 'mail2.example.com.',
+                            'priority': 2
+                        }
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 0,
+                "records": []
+            }
+            ExpectChanges = True
+            ExpectedAdditions = {
+                "records": [{
+                    "name": "foo.unit.tests",
+                    "type": "MX",
+                    "data": "mail2.example.com",
+                    "priority": 2,
+                    "ttl": 300
+                }, {
+                    "name": "unit.tests",
+                    "type": "MX",
+                    "data": "mail1.example.com",
+                    "priority": 1,
+                    "ttl": 300
+                }]
+            }
+            ExpectedDeletions = None
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_multiple_additions_splatting(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": '',
+                    "data": {
+                        'type': 'A',
+                        'ttl': 300,
+                        'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6']
+                    }
+                },
+                {
+                    "subdomain": 'foo',
+                    "data": {
+                        'type': 'NS',
+                        'ttl': 300,
+                        'values': ['ns1.example.com.', 'ns2.example.com.']
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 0,
+                "records": []
+            }
+            ExpectChanges = True
+            ExpectedAdditions = {
+                "records": [{
+                    "name": "unit.tests",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }, {
+                    "name": "unit.tests",
+                    "type": "A",
+                    "data": "1.2.3.5",
+                    "ttl": 300
+                }, {
+                    "name": "unit.tests",
+                    "type": "A",
+                    "data": "1.2.3.6",
+                    "ttl": 300
+                }, {
+                    "name": "foo.unit.tests",
+                    "type": "NS",
+                    "data": "ns1.example.com",
+                    "ttl": 300
+                }, {
+                    "name": "foo.unit.tests",
+                    "type": "NS",
+                    "data": "ns2.example.com",
+                    "ttl": 300
+                }]
+            }
+            ExpectedDeletions = None
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_multiple_additions_namespaced(self):
+        class TestData(object):
+            OtherRecords = [{
+                "subdomain": 'foo',
+                "data": {
+                    'type': 'A',
+                    'ttl': 300,
+                    'value': '1.2.3.4'
+                }
+            }, {
+                "subdomain": 'bar',
+                "data": {
+                    'type': 'A',
+                    'ttl': 300,
+                    'value': '1.2.3.4'
+                }
+            }, {
+                "subdomain": 'foo',
+                "data": {
+                    'type': 'NS',
+                    'ttl': 300,
+                    'value': 'ns.example.com.'
+                }
+            }]
+            OwnRecords = {
+                "totalEntries": 0,
+                "records": []
+            }
+            ExpectChanges = True
+            ExpectedAdditions = {
+                "records": [{
+                    "name": "bar.unit.tests",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }, {
+                    "name": "foo.unit.tests",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }, {
+                    "name": "foo.unit.tests",
+                    "type": "NS",
+                    "data": "ns.example.com",
+                    "ttl": 300
+                }]
+            }
+            ExpectedDeletions = None
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_single_deletion(self):
+        class TestData(object):
+            OtherRecords = []
+            OwnRecords = {
+                "totalEntries": 1,
+                "records": [{
+                    "name": "unit.tests",
+                    "id": "A-111111",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }, {
+                    "name": "foo.unit.tests",
+                    "id": "NS-111111",
+                    "type": "NS",
+                    "data": "ns.example.com",
+                    "ttl": 300
+                }]
+            }
+            ExpectChanges = True
+            ExpectedAdditions = None
+            ExpectedDeletions = "id=A-111111&id=NS-111111"
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_multiple_deletions(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": '',
+                    "data": {
+                        'type': 'A',
+                        'ttl': 300,
+                        'value': '1.2.3.5'
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 3,
+                "records": [{
+                    "name": "unit.tests",
+                    "id": "A-111111",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }, {
+                    "name": "unit.tests",
+                    "id": "A-222222",
+                    "type": "A",
+                    "data": "1.2.3.5",
+                    "ttl": 300
+                }, {
+                    "name": "unit.tests",
+                    "id": "A-333333",
+                    "type": "A",
+                    "data": "1.2.3.6",
+                    "ttl": 300
+                }, {
+                    "name": "foo.unit.tests",
+                    "id": "NS-111111",
+                    "type": "NS",
+                    "data": "ns.example.com",
+                    "ttl": 300
+                }]
+            }
+            ExpectChanges = True
+            ExpectedAdditions = None
+            ExpectedDeletions = "id=A-111111&id=A-333333&id=NS-111111"
+            ExpectedUpdates = {
+                "records": [{
+                    "name": "unit.tests",
+                    "id": "A-222222",
+                    "data": "1.2.3.5",
+                    "ttl": 300
+                }]
+            }
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_multiple_deletions_cross_zone(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": '',
+                    "data": {
+                        'type': 'A',
+                        'ttl': 300,
+                        'value': '1.2.3.4'
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 3,
+                "records": [{
+                    "name": "unit.tests",
+                    "id": "A-111111",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }, {
+                    "name": "foo.unit.tests",
+                    "id": "A-222222",
+                    "type": "A",
+                    "data": "1.2.3.5",
+                    "ttl": 300
+                }, {
+                    "name": "bar.unit.tests",
+                    "id": "A-333333",
+                    "type": "A",
+                    "data": "1.2.3.6",
+                    "ttl": 300
+                }]
+            }
+            ExpectChanges = True
+            ExpectedAdditions = None
+            ExpectedDeletions = "id=A-222222&id=A-333333"
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_delete_cname(self):
+        class TestData(object):
+            OtherRecords = []
+            OwnRecords = {
+                "totalEntries": 3,
+                "records": [{
+                    "name": "foo.unit.tests",
+                    "id": "CNAME-111111",
+                    "type": "CNAME",
+                    "data": "a.example.com",
+                    "ttl": 300
+                }]
+            }
+            ExpectChanges = True
+            ExpectedAdditions = None
+            ExpectedDeletions = "id=CNAME-111111"
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_single_update(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": '',
+                    "data": {
+                        'type': 'A',
+                        'ttl': 3600,
+                        'value': '1.2.3.4'
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 1,
+                "records": [{
+                    "name": "unit.tests",
+                    "id": "A-111111",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }]
+            }
+            ExpectChanges = True
+            ExpectedAdditions = None
+            ExpectedDeletions = None
+            ExpectedUpdates = {
+                "records": [{
+                    "name": "unit.tests",
+                    "id": "A-111111",
+                    "data": "1.2.3.4",
+                    "ttl": 3600
+                }]
+            }
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_update_TXT(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": '',
+                    "data": {
+                        'type': 'TXT',
+                        'ttl': 300,
+                        'value': 'othervalue'
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 1,
+                "records": [{
+                    "name": "unit.tests",
+                    "id": "TXT-111111",
+                    "type": "TXT",
+                    "data": "somevalue",
+                    "ttl": 300
+                }]
+            }
+            ExpectChanges = True
+            ExpectedAdditions = {
+                "records": [{
+                    "name": "unit.tests",
+                    "type": "TXT",
+                    "data": "othervalue",
+                    "ttl": 300
+                }]
+            }
+            ExpectedDeletions = 'id=TXT-111111'
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_update_MX(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": '',
+                    "data": {
+                        'type': 'MX',
+                        'ttl': 300,
+                        'value': {u'priority': 50, u'value': 'mx.test.com.'}
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 1,
+                "records": [{
+                    "name": "unit.tests",
+                    "id": "MX-111111",
+                    "type": "MX",
+                    "priority": 20,
+                    "data": "mx.test.com",
+                    "ttl": 300
+                }]
+            }
+            ExpectChanges = True
+            ExpectedAdditions = {
+                "records": [{
+                    "name": "unit.tests",
+                    "type": "MX",
+                    "priority": 50,
+                    "data": "mx.test.com",
+                    "ttl": 300
+                }]
+            }
+            ExpectedDeletions = 'id=MX-111111'
+            ExpectedUpdates = None
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_multiple_updates(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": '',
+                    "data": {
+                        'type': 'A',
+                        'ttl': 3600,
+                        'values': ['1.2.3.4', '1.2.3.5', '1.2.3.6']
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 3,
+                "records": [{
+                    "name": "unit.tests",
+                    "id": "A-111111",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }, {
+                    "name": "unit.tests",
+                    "id": "A-222222",
+                    "type": "A",
+                    "data": "1.2.3.5",
+                    "ttl": 300
+                }, {
+                    "name": "unit.tests",
+                    "id": "A-333333",
+                    "type": "A",
+                    "data": "1.2.3.6",
+                    "ttl": 300
+                }]
+            }
+            ExpectChanges = True
+            ExpectedAdditions = None
+            ExpectedDeletions = None
+            ExpectedUpdates = {
+                "records": [{
+                    "name": "unit.tests",
+                    "id": "A-222222",
+                    "data": "1.2.3.5",
+                    "ttl": 3600
+                }, {
+                    "name": "unit.tests",
+                    "id": "A-111111",
+                    "data": "1.2.3.4",
+                    "ttl": 3600
+                }, {
+                    "name": "unit.tests",
+                    "id": "A-333333",
+                    "data": "1.2.3.6",
+                    "ttl": 3600
+                }]
+            }
+
+        return self._test_apply_with_data(TestData)
+
+    def test_apply_multiple_updates_cross_zone(self):
+        class TestData(object):
+            OtherRecords = [
+                {
+                    "subdomain": 'foo',
+                    "data": {
+                        'type': 'A',
+                        'ttl': 3600,
+                        'value': '1.2.3.4'
+                    }
+                },
+                {
+                    "subdomain": 'bar',
+                    "data": {
+                        'type': 'A',
+                        'ttl': 3600,
+                        'value': '1.2.3.4'
+                    }
+                }
+            ]
+            OwnRecords = {
+                "totalEntries": 2,
+                "records": [{
+                    "name": "foo.unit.tests",
+                    "id": "A-111111",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }, {
+                    "name": "bar.unit.tests",
+                    "id": "A-222222",
+                    "type": "A",
+                    "data": "1.2.3.4",
+                    "ttl": 300
+                }]
+            }
+            ExpectChanges = True
+            ExpectedAdditions = None
+            ExpectedDeletions = None
+            ExpectedUpdates = {
+                "records": [{
+                    "name": "bar.unit.tests",
+                    "id": "A-222222",
+                    "data": "1.2.3.4",
+                    "ttl": 3600
+                }, {
+                    "name": "foo.unit.tests",
+                    "id": "A-111111",
+                    "data": "1.2.3.4",
+                    "ttl": 3600
+                }]
+            }
+
+        return self._test_apply_with_data(TestData)