Merge branch 'ns1_geo' of ssh://github.com/nsone/octodns into ns1_geo

CONTRIBUTING.md

@@ -38,6 +38,10 @@ Here are a few things you can do that will increase the likelihood of your pull
 
 - Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
 
+## Development prerequisites
+
+- setuptools >= 30.3.0
+
 ## License note
 
 We can only accept contributions that are compatible with the MIT license.

MANIFEST.in

@@ -3,6 +3,5 @@ include CONTRIBUTING.md
 include LICENSE
 include docs/*
 include octodns/*
-include requirements*.txt
 include script/*
 include tests/*

README.md

@@ -151,6 +151,7 @@ The above command pulled the existing data out of Route53 and placed the results
 |--|--|--|--|
 | [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | |
 | [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted |
+| [DigitalOceanProvider](/octodns/provider/digitalocean.py) | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted |
 | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted |
 | [DynProvider](/octodns/provider/dyn.py) | All | Yes | |
 | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT  | No | |

docs/records.md

@@ -26,6 +26,55 @@ GeoDNS is currently supported for `A` and `AAAA` records on the Dyn (via Traffic
 
 Configuring GeoDNS is complex and the details of the functionality vary widely from provider to provider. OctoDNS has an opinionated view of how GeoDNS should be set up and does its best to map that to each provider's offering in a way that will result in similar behavior. It may not fit your needs or use cases, in which case please open an issue for discussion. We expect this functionality to grow and evolve over time as it's more widely used.
 
+The following is an example of GeoDNS with three entries NA-US-CA, NA-US-NY, OC-AU. Octodns creates another one labeled 'default' with the details for the actual A record, This default record is the failover record if the monitoring check fails.
+
+```yaml
+---
+? ''
+: type: TXT
+  value: v=spf1 -all
+test:
+  geo:
+    NA-US-NY:
+    - 111.111.111.1
+    NA-US-CA:
+    - 111.111.111.2
+    OC-AU:
+    - 111.111.111.3
+    EU:
+    - 111.111.111.4
+  ttl: 300
+  type: A
+  value: 111.111.111.5
+```
+
+
+The geo labels breakdown based on:
+
+1.
+    - 'AF': 14,  # Continental Africa
+    - 'AN': 17,  # Continental Antartica
+    - 'AS': 15,  # Contentinal Asia
+    - 'EU': 13,  # Contentinal Europe
+    - 'NA': 11,  # Continental North America
+    - 'OC': 16,  # Contentinal Austrailia/Oceania
+    - 'SA': 12,  # Continental South America
+
+2. ISO Country Code https://en.wikipedia.org/wiki/ISO_3166-2
+
+3. ISO Country Code Subdevision as per https://en.wikipedia.org/wiki/ISO_3166-2:US   (change the code at the end for the country you are subdividing) * these may not always be supported depending on the provider.
+
+So the example is saying:
+
+- North America - United States - New York:  gets served an "A" record of  111.111.111.1
+- North America - United States - California:   gets served an "A" record of  111.111.111.2
+- Oceania - Australia: Gets served an "A" record of 111.111.111.3
+- Europe: gets an "A" record of 111.111.111.4
+- Everyone else gets an "A" record of 111.111.111.5
+
+
+Octodns will automatically set up a monitor and check for **https://<ip_address>/_dns** and check for a 200 response.
+
 ## Config (`YamlProvider`)
 
 OctoDNS records and `YamlProvider`'s schema is essentially a 1:1 match. Properties on the objects will match keys in the config.

octodns/cmds/validate.py

@@ -15,7 +15,7 @@ from octodns.manager import Manager
 def main():
     parser = ArgumentParser(description=__doc__.split('\n')[1])
 
-    parser.add_argument('--config-file', default='./config/production.yaml',
+    parser.add_argument('--config-file', required=True,
                         help='The Manager configuration file to use')
 
     args = parser.parse_args(WARN)

octodns/provider/digitalocean.py

@@ -0,0 +1,336 @@
+#
+#
+#
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from collections import defaultdict
+from requests import Session
+import logging
+
+from ..record import Record
+from .base import BaseProvider
+
+
+class DigitalOceanClientException(Exception):
+    pass
+
+
+class DigitalOceanClientNotFound(DigitalOceanClientException):
+
+    def __init__(self):
+        super(DigitalOceanClientNotFound, self).__init__('Not Found')
+
+
+class DigitalOceanClientUnauthorized(DigitalOceanClientException):
+
+    def __init__(self):
+        super(DigitalOceanClientUnauthorized, self).__init__('Unauthorized')
+
+
+class DigitalOceanClient(object):
+    BASE = 'https://api.digitalocean.com/v2'
+
+    def __init__(self, token):
+        sess = Session()
+        sess.headers.update({'Authorization': 'Bearer {}'.format(token)})
+        self._sess = sess
+
+    def _request(self, method, path, params=None, data=None):
+        url = '{}{}'.format(self.BASE, path)
+        resp = self._sess.request(method, url, params=params, json=data)
+        if resp.status_code == 401:
+            raise DigitalOceanClientUnauthorized()
+        if resp.status_code == 404:
+            raise DigitalOceanClientNotFound()
+        resp.raise_for_status()
+        return resp
+
+    def domain(self, name):
+        path = '/domains/{}'.format(name)
+        return self._request('GET', path).json()
+
+    def domain_create(self, name):
+        # Digitalocean requires an IP on zone creation
+        self._request('POST', '/domains', data={'name': name,
+                                                'ip_address': '192.0.2.1'})
+
+        # After the zone is created, immeadiately delete the record
+        records = self.records(name)
+        for record in records:
+            if record['name'] == '' and record['type'] == 'A':
+                self.record_delete(name, record['id'])
+
+    def records(self, zone_name):
+        path = '/domains/{}/records'.format(zone_name)
+        ret = []
+
+        page = 1
+        while True:
+            data = self._request('GET', path, {'page': page}).json()
+
+            ret += data['domain_records']
+            links = data['links']
+
+            # https://developers.digitalocean.com/documentation/v2/#links
+            # pages exists if there is more than 1 page
+            # last doesn't exist if you're on the last page
+            try:
+                links['pages']['last']
+                page += 1
+            except KeyError:
+                break
+
+        # change any apex record to empty string to match other provider output
+        for record in ret:
+            if record['name'] == '@':
+                record['name'] = ''
+
+        return ret
+
+    def record_create(self, zone_name, params):
+        path = '/domains/{}/records'.format(zone_name)
+        # change empty string to @, DigitalOcean uses @ for apex record names
+        if params['name'] == '':
+            params['name'] = '@'
+        self._request('POST', path, data=params)
+
+    def record_delete(self, zone_name, record_id):
+        path = '/domains/{}/records/{}'.format(zone_name, record_id)
+        self._request('DELETE', path)
+
+
+class DigitalOceanProvider(BaseProvider):
+    '''
+    DigitalOcean DNS provider using API v2
+
+    digitalocean:
+        class: octodns.provider.digitalocean.DigitalOceanProvider
+        # Your DigitalOcean API token (required)
+        token: foo
+    '''
+    SUPPORTS_GEO = False
+    SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'TXT', 'SRV'))
+
+    def __init__(self, id, token, *args, **kwargs):
+        self.log = logging.getLogger('DigitalOceanProvider[{}]'.format(id))
+        self.log.debug('__init__: id=%s, token=***', id)
+        super(DigitalOceanProvider, self).__init__(id, *args, **kwargs)
+        self._client = DigitalOceanClient(token)
+
+        self._zone_records = {}
+
+    def _data_for_multiple(self, _type, records):
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': [r['data'] for r in records]
+        }
+
+    _data_for_A = _data_for_multiple
+    _data_for_AAAA = _data_for_multiple
+
+    def _data_for_CAA(self, _type, records):
+        values = []
+        for record in records:
+            values.append({
+                'flags': record['flags'],
+                'tag': record['tag'],
+                'value': record['data'],
+            })
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values
+        }
+
+    def _data_for_CNAME(self, _type, records):
+        record = records[0]
+        return {
+            'ttl': record['ttl'],
+            'type': _type,
+            'value': '{}.'.format(record['data'])
+        }
+
+    def _data_for_MX(self, _type, records):
+        values = []
+        for record in records:
+            values.append({
+                'preference': record['priority'],
+                'exchange': '{}.'.format(record['data'])
+            })
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values
+        }
+
+    def _data_for_NS(self, _type, records):
+        values = []
+        for record in records:
+            data = '{}.'.format(record['data'])
+            values.append(data)
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values,
+        }
+
+    def _data_for_SRV(self, _type, records):
+        values = []
+        for record in records:
+            values.append({
+                'port': record['port'],
+                'priority': record['priority'],
+                'target': '{}.'.format(record['data']),
+                'weight': record['weight']
+            })
+        return {
+            'type': _type,
+            'ttl': records[0]['ttl'],
+            'values': values
+        }
+
+    def _data_for_TXT(self, _type, records):
+        values = [value['data'].replace(';', '\;') for value in records]
+        return {
+            'ttl': records[0]['ttl'],
+            'type': _type,
+            'values': values
+        }
+
+    def zone_records(self, zone):
+        if zone.name not in self._zone_records:
+            try:
+                self._zone_records[zone.name] = \
+                    self._client.records(zone.name[:-1])
+            except DigitalOceanClientNotFound:
+                return []
+
+        return self._zone_records[zone.name]
+
+    def populate(self, zone, target=False, lenient=False):
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+
+        values = defaultdict(lambda: defaultdict(list))
+        for record in self.zone_records(zone):
+            _type = record['type']
+            values[record['name']][record['type']].append(record)
+
+        before = len(zone.records)
+        for name, types in values.items():
+            for _type, records in types.items():
+                data_for = getattr(self, '_data_for_{}'.format(_type))
+                record = Record.new(zone, name, data_for(_type, records),
+                                    source=self, lenient=lenient)
+                zone.add_record(record)
+
+        self.log.info('populate:   found %s records',
+                      len(zone.records) - before)
+
+    def _params_for_multiple(self, record):
+        for value in record.values:
+            yield {
+                'data': value,
+                'name': record.name,
+                'ttl': record.ttl,
+                'type': record._type
+            }
+
+    _params_for_A = _params_for_multiple
+    _params_for_AAAA = _params_for_multiple
+    _params_for_NS = _params_for_multiple
+
+    def _params_for_CAA(self, record):
+        for value in record.values:
+            yield {
+                'data': '{}.'.format(value.value),
+                'flags': value.flags,
+                'name': record.name,
+                'tag': value.tag,
+                'ttl': record.ttl,
+                'type': record._type
+            }
+
+    def _params_for_single(self, record):
+        yield {
+            'data': record.value,
+            'name': record.name,
+            'ttl': record.ttl,
+            'type': record._type
+        }
+
+    _params_for_CNAME = _params_for_single
+
+    def _params_for_MX(self, record):
+        for value in record.values:
+            yield {
+                'data': value.exchange,
+                'name': record.name,
+                'priority': value.preference,
+                'ttl': record.ttl,
+                'type': record._type
+            }
+
+    def _params_for_SRV(self, record):
+        for value in record.values:
+            yield {
+                'data': value.target,
+                'name': record.name,
+                'port': value.port,
+                'priority': value.priority,
+                'ttl': record.ttl,
+                'type': record._type,
+                'weight': value.weight
+            }
+
+    def _params_for_TXT(self, record):
+        # DigitalOcean doesn't want things escaped in values so we
+        # have to strip them here and add them when going the other way
+        for value in record.values:
+            yield {
+                'data': value.replace('\;', ';'),
+                'name': record.name,
+                'ttl': record.ttl,
+                'type': record._type
+            }
+
+    def _apply_Create(self, change):
+        new = change.new
+        params_for = getattr(self, '_params_for_{}'.format(new._type))
+        for params in params_for(new):
+            self._client.record_create(new.zone.name[:-1], params)
+
+    def _apply_Update(self, change):
+        self._apply_Delete(change)
+        self._apply_Create(change)
+
+    def _apply_Delete(self, change):
+        existing = change.existing
+        zone = existing.zone
+        for record in self.zone_records(zone):
+            if existing.name == record['name'] and \
+               existing._type == record['type']:
+                self._client.record_delete(zone.name[:-1], record['id'])
+
+    def _apply(self, plan):
+        desired = plan.desired
+        changes = plan.changes
+        self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name,
+                       len(changes))
+
+        domain_name = desired.name[:-1]
+        try:
+            self._client.domain(domain_name)
+        except DigitalOceanClientNotFound:
+            self.log.debug('_apply:   no matching zone, creating domain')
+            self._client.domain_create(domain_name)
+
+        for change in changes:
+            class_name = change.__class__.__name__
+            getattr(self, '_apply_{}'.format(class_name))(change)
+
+        # Clear out the cache if any
+        self._zone_records.pop(desired.name, None)

octodns/provider/powerdns.py

@@ -18,13 +18,14 @@ class PowerDnsBaseProvider(BaseProvider):
                     'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT'))
     TIMEOUT = 5
 
-    def __init__(self, id, host, api_key, port=8081, scheme="http", *args,
+    def __init__(self, id, host, api_key, port=8081, scheme="http",
-                 **kwargs):
+                 timeout=TIMEOUT, *args, **kwargs):
         super(PowerDnsBaseProvider, self).__init__(id, *args, **kwargs)
 
         self.host = host
         self.port = port
         self.scheme = scheme
+        self.timeout = timeout
 
         sess = Session()
         sess.headers.update({'X-API-Key': api_key})
@@ -35,7 +36,7 @@ class PowerDnsBaseProvider(BaseProvider):
 
         url = '{}://{}:{}/api/v1/servers/localhost/{}' \
             .format(self.scheme, self.host, self.port, path)
-        resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT)
+        resp = self._sess.request(method, url, json=data, timeout=self.timeout)
         self.log.debug('_request:   status=%d', resp.status_code)
         resp.raise_for_status()
         return resp

octodns/provider/yaml.py

@@ -31,8 +31,8 @@ class YamlProvider(BaseProvider):
         enforce_order: True
     '''
     SUPPORTS_GEO = True
-    SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR',
+    SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS',
-                   'SSHFP', 'SPF', 'SRV', 'TXT'))
+                    'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT'))
 
     def __init__(self, id, directory, default_ttl=3600, enforce_order=True,
                  *args, **kwargs):

requirements-dev.txt

@@ -1,7 +0,0 @@
-coverage
-mock
-nose
-pep8
-pyflakes
-requests_mock
-setuptools>=36.4.0

requirements.txt

@@ -1,23 +0,0 @@
-# These are known good versions. You're free to use others and things will
-# likely work, but no promises are made, especilly if you go older.
-PyYaml==3.12
-azure-mgmt-dns==1.0.1
-azure-common==1.1.6
-boto3==1.4.6
-botocore==1.6.8
-dnspython==1.15.0
-docutils==0.14
-dyn==1.8.0
-futures==3.1.1
-google-cloud==0.27.0
-incf.countryutils==1.0
-ipaddress==1.0.18
-jmespath==0.9.3
-msrestazure==0.4.10
-natsort==5.0.3
-nsone==0.9.14
-ovh==0.4.7
-python-dateutil==2.6.1
-requests==2.13.0
-s3transfer==0.1.10
-six==1.10.0

script/bootstrap

@@ -19,10 +19,10 @@ if [ ! -d "$VENV_NAME" ]; then
 fi
 . "$VENV_NAME/bin/activate"
 
-pip install -U -r requirements.txt
+pip install -e .
 
 if [ "$ENV" != "production" ]; then
-    pip install -U -r requirements-dev.txt
+    pip install -e .[dev,test]
 fi
 
 if [ ! -L ".git/hooks/pre-commit" ]; then

setup.cfg

@@ -0,0 +1,68 @@
+[metadata]
+name = octodns
+description = "DNS as code - Tools for managing DNS across multiple providers"
+long_description = file: README.md
+version = attr: octodns.__VERSION__
+author = Ross McFarland
+author_email = rwmcfa1@gmail.com
+url = https://github.com/github/octodns
+license = MIT
+keywords = dns, providers
+classifiers =
+    License :: OSI Approved :: MIT License
+    Programming Language :: Python
+    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.3
+    Programming Language :: Python :: 3.4
+    Programming Language :: Python :: 3.5
+    Programming Language :: Python :: 3.6
+
+[options]
+install_requires = 
+    PyYaml>=3.12
+    dnspython>=1.15.0
+    futures==3.1.1
+    incf.countryutils==1.0
+    ipaddress==1.0.18
+    natsort==5.0.3
+    python-dateutil==2.6.1
+    requests==2.13.0
+packages = find:
+include_package_data = True
+
+[options.entry_points]
+console_scripts = 
+    octodns-compare = octodns.cmds.compare:main
+    octodns-dump = octodns.cmds.dump:main
+    octodns-report = octodns.cmds.report:main
+    octodns-sync = octodns.cmds.sync:main
+    octodns-validate = octodns.cmds.validate:main
+
+[options.packages.find]
+exclude =
+    tests
+
+[options.extras_require]
+dev = 
+    azure-mgmt-dns==1.0.1
+    azure-common==1.1.6
+    boto3==1.4.6
+    botocore==1.6.8
+    docutils==0.14
+    dyn==1.8.0
+    google-cloud==0.27.0
+    jmespath==0.9.3
+    msrestazure==0.4.10
+    nsone==0.9.14
+    ovh==0.4.7
+    s3transfer==0.1.10
+    six==1.10.0
+test =
+    coverage
+    mock
+    nose
+    pep8
+    pyflakes
+    requests_mock
+    setuptools>=36.4.0

setup.py

@@ -1,47 +1,5 @@
 #!/usr/bin/env python
+from setuptools import setup
 
-from os.path import dirname, join
-import octodns
 
-try:
+setup()
-    from setuptools import find_packages, setup
-except ImportError:
-    from distutils.core import find_packages, setup
-
-cmds = (
-    'compare',
-    'dump',
-    'report',
-    'sync',
-    'validate'
-)
-cmds_dir = join(dirname(__file__), 'octodns', 'cmds')
-console_scripts = {
-    'octodns-{name} = octodns.cmds.{name}:main'.format(name=name)
-    for name in cmds
-}
-
-setup(
-    author='Ross McFarland',
-    author_email='rwmcfa1@gmail.com',
-    description=octodns.__doc__,
-    entry_points={
-        'console_scripts': console_scripts,
-    },
-    install_requires=[
-        'PyYaml>=3.12',
-        'dnspython>=1.15.0',
-        'futures>=3.0.5',
-        'incf.countryutils>=1.0',
-        'ipaddress>=1.0.18',
-        'natsort>=5.0.3',
-        'python-dateutil>=2.6.0',
-        'requests>=2.13.0'
-    ],
-    license='MIT',
-    long_description=open('README.md').read(),
-    name='octodns',
-    packages=find_packages(),
-    url='https://github.com/github/octodns',
-    version=octodns.__VERSION__,
-)

tests/fixtures/digitalocean-page-1.json

@@ -0,0 +1,177 @@
+{
+	"domain_records": [{
+		"id": 11189874,
+		"type": "NS",
+		"name": "@",
+		"data": "ns1.digitalocean.com",
+		"priority": null,
+		"port": null,
+		"ttl": 3600,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189875,
+		"type": "NS",
+		"name": "@",
+		"data": "ns2.digitalocean.com",
+		"priority": null,
+		"port": null,
+		"ttl": 3600,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189876,
+		"type": "NS",
+		"name": "@",
+		"data": "ns3.digitalocean.com",
+		"priority": null,
+		"port": null,
+		"ttl": 3600,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189877,
+		"type": "NS",
+		"name": "under",
+		"data": "ns1.unit.tests",
+		"priority": null,
+		"port": null,
+		"ttl": 3600,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189878,
+		"type": "NS",
+		"name": "under",
+		"data": "ns2.unit.tests",
+		"priority": null,
+		"port": null,
+		"ttl": 3600,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189879,
+		"type": "SRV",
+		"name": "_srv._tcp",
+		"data": "foo-1.unit.tests",
+		"priority": 10,
+		"port": 30,
+		"ttl": 600,
+		"weight": 20,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189880,
+		"type": "SRV",
+		"name": "_srv._tcp",
+		"data": "foo-2.unit.tests",
+		"priority": 12,
+		"port": 30,
+		"ttl": 600,
+		"weight": 20,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189881,
+		"type": "TXT",
+		"name": "txt",
+		"data": "Bah bah black sheep",
+		"priority": null,
+		"port": null,
+		"ttl": 600,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189882,
+		"type": "TXT",
+		"name": "txt",
+		"data": "have you any wool.",
+		"priority": null,
+		"port": null,
+		"ttl": 600,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189883,
+		"type": "A",
+		"name": "@",
+		"data": "1.2.3.4",
+		"priority": null,
+		"port": null,
+		"ttl": 300,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189884,
+		"type": "A",
+		"name": "@",
+		"data": "1.2.3.5",
+		"priority": null,
+		"port": null,
+		"ttl": 300,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189885,
+		"type": "A",
+		"name": "www",
+		"data": "2.2.3.6",
+		"priority": null,
+		"port": null,
+		"ttl": 300,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189886,
+		"type": "MX",
+		"name": "mx",
+		"data": "smtp-4.unit.tests",
+		"priority": 10,
+		"port": null,
+		"ttl": 300,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189887,
+		"type": "MX",
+		"name": "mx",
+		"data": "smtp-2.unit.tests",
+		"priority": 20,
+		"port": null,
+		"ttl": 300,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189888,
+		"type": "MX",
+		"name": "mx",
+		"data": "smtp-3.unit.tests",
+		"priority": 30,
+		"port": null,
+		"ttl": 300,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}],
+	"links": {
+		"pages": {
+			"last": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=2",
+			"next": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=2"
+		}
+	},
+	"meta": {
+		"total": 21
+	}
+}

tests/fixtures/digitalocean-page-2.json

@@ -0,0 +1,89 @@
+{
+	"domain_records": [{
+		"id": 11189889,
+		"type": "MX",
+		"name": "mx",
+		"data": "smtp-1.unit.tests",
+		"priority": 40,
+		"port": null,
+		"ttl": 300,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189890,
+		"type": "AAAA",
+		"name": "aaaa",
+		"data": "2601:644:500:e210:62f8:1dff:feb8:947a",
+		"priority": null,
+		"port": null,
+		"ttl": 600,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189891,
+		"type": "CNAME",
+		"name": "cname",
+		"data": "unit.tests",
+		"priority": null,
+		"port": null,
+		"ttl": 300,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189892,
+		"type": "A",
+		"name": "www.sub",
+		"data": "2.2.3.6",
+		"priority": null,
+		"port": null,
+		"ttl": 300,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189893,
+		"type": "TXT",
+		"name": "txt",
+		"data": "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs",
+		"priority": null,
+		"port": null,
+		"ttl": 600,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}, {
+		"id": 11189894,
+		"type": "CAA",
+		"name": "@",
+		"data": "ca.unit.tests",
+		"priority": null,
+		"port": null,
+		"ttl": 3600,
+		"weight": null,
+		"flags": 0,
+		"tag": "issue"
+	}, {
+		"id": 11189895,
+		"type": "CNAME",
+		"name": "included",
+		"data": "unit.tests",
+		"priority": null,
+		"port": null,
+		"ttl": 3600,
+		"weight": null,
+		"flags": null,
+		"tag": null
+	}],
+	"links": {
+		"pages": {
+			"first": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=1",
+			"prev": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=1"
+		}
+	},
+	"meta": {
+		"total": 21
+	}
+}

tests/test_octodns_manager.py

@@ -102,12 +102,12 @@ class TestManager(TestCase):
             environ['YAML_TMP_DIR'] = tmpdir.dirname
             tc = Manager(get_config_filename('simple.yaml')) \
                 .sync(dry_run=False)
-            self.assertEquals(20, tc)
+            self.assertEquals(21, tc)
 
             # try with just one of the zones
             tc = Manager(get_config_filename('simple.yaml')) \
                 .sync(dry_run=False, eligible_zones=['unit.tests.'])
-            self.assertEquals(14, tc)
+            self.assertEquals(15, tc)
 
             # the subzone, with 2 targets
             tc = Manager(get_config_filename('simple.yaml')) \
@@ -122,18 +122,18 @@ class TestManager(TestCase):
             # Again with force
             tc = Manager(get_config_filename('simple.yaml')) \
                 .sync(dry_run=False, force=True)
-            self.assertEquals(20, tc)
+            self.assertEquals(21, tc)
 
             # Again with max_workers = 1
             tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \
                 .sync(dry_run=False, force=True)
-            self.assertEquals(20, tc)
+            self.assertEquals(21, tc)
 
             # Include meta
             tc = Manager(get_config_filename('simple.yaml'), max_workers=1,
                          include_meta=True) \
                 .sync(dry_run=False, force=True)
-            self.assertEquals(24, tc)
+            self.assertEquals(25, tc)
 
     def test_eligible_targets(self):
         with TemporaryDirectory() as tmpdir:
@@ -159,13 +159,13 @@ class TestManager(TestCase):
                 fh.write('---\n{}')
 
             changes = manager.compare(['in'], ['dump'], 'unit.tests.')
-            self.assertEquals(14, len(changes))
+            self.assertEquals(15, len(changes))
 
             # Compound sources with varying support
             changes = manager.compare(['in', 'nosshfp'],
                                       ['dump'],
                                       'unit.tests.')
-            self.assertEquals(13, len(changes))
+            self.assertEquals(14, len(changes))
 
             with self.assertRaises(Exception) as ctx:
                 manager.compare(['nope'], ['dump'], 'unit.tests.')

tests/test_octodns_provider_digitalocean.py

@@ -0,0 +1,241 @@
+#
+#
+#
+
+
+from __future__ import absolute_import, division, print_function, \
+    unicode_literals
+
+from mock import Mock, call
+from os.path import dirname, join
+from requests import HTTPError
+from requests_mock import ANY, mock as requests_mock
+from unittest import TestCase
+
+from octodns.record import Record
+from octodns.provider.digitalocean import DigitalOceanClientNotFound, \
+    DigitalOceanProvider
+from octodns.provider.yaml import YamlProvider
+from octodns.zone import Zone
+
+
+class TestDigitalOceanProvider(TestCase):
+    expected = Zone('unit.tests.', [])
+    source = YamlProvider('test', join(dirname(__file__), 'config'))
+    source.populate(expected)
+
+    # Our test suite differs a bit, add our NS and remove the simple one
+    expected.add_record(Record.new(expected, 'under', {
+        'ttl': 3600,
+        'type': 'NS',
+        'values': [
+            'ns1.unit.tests.',
+            'ns2.unit.tests.',
+        ]
+    }))
+    for record in list(expected.records):
+        if record.name == 'sub' and record._type == 'NS':
+            expected._remove_record(record)
+            break
+
+    def test_populate(self):
+        provider = DigitalOceanProvider('test', 'token')
+
+        # Bad auth
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=401,
+                     text='{"id":"unauthorized",'
+                     '"message":"Unable to authenticate you."}')
+
+            with self.assertRaises(Exception) as ctx:
+                zone = Zone('unit.tests.', [])
+                provider.populate(zone)
+            self.assertEquals('Unauthorized', ctx.exception.message)
+
+        # General error
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=502, text='Things caught fire')
+
+            with self.assertRaises(HTTPError) as ctx:
+                zone = Zone('unit.tests.', [])
+                provider.populate(zone)
+            self.assertEquals(502, ctx.exception.response.status_code)
+
+        # Non-existant zone doesn't populate anything
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=404,
+                     text='{"id":"not_found","message":"The resource you '
+                     'were accessing could not be found."}')
+
+            zone = Zone('unit.tests.', [])
+            provider.populate(zone)
+            self.assertEquals(set(), zone.records)
+
+        # No diffs == no changes
+        with requests_mock() as mock:
+            base = 'https://api.digitalocean.com/v2/domains/unit.tests/' \
+                'records?page='
+            with open('tests/fixtures/digitalocean-page-1.json') as fh:
+                mock.get('{}{}'.format(base, 1), text=fh.read())
+            with open('tests/fixtures/digitalocean-page-2.json') as fh:
+                mock.get('{}{}'.format(base, 2), text=fh.read())
+
+            zone = Zone('unit.tests.', [])
+            provider.populate(zone)
+            self.assertEquals(12, len(zone.records))
+            changes = self.expected.changes(zone, provider)
+            self.assertEquals(0, len(changes))
+
+        # 2nd populate makes no network calls/all from cache
+        again = Zone('unit.tests.', [])
+        provider.populate(again)
+        self.assertEquals(12, len(again.records))
+
+        # bust the cache
+        del provider._zone_records[zone.name]
+
+    def test_apply(self):
+        provider = DigitalOceanProvider('test', 'token')
+
+        resp = Mock()
+        resp.json = Mock()
+        provider._client._request = Mock(return_value=resp)
+
+        domain_after_creation = {
+            "domain_records": [{
+                "id": 11189874,
+                "type": "NS",
+                "name": "@",
+                "data": "ns1.digitalocean.com",
+                "priority": None,
+                "port": None,
+                "ttl": 3600,
+                "weight": None,
+                "flags": None,
+                "tag": None
+            }, {
+                "id": 11189875,
+                "type": "NS",
+                "name": "@",
+                "data": "ns2.digitalocean.com",
+                "priority": None,
+                "port": None,
+                "ttl": 3600,
+                "weight": None,
+                "flags": None,
+                "tag": None
+            }, {
+                "id": 11189876,
+                "type": "NS",
+                "name": "@",
+                "data": "ns3.digitalocean.com",
+                "priority": None,
+                "port": None,
+                "ttl": 3600,
+                "weight": None,
+                "flags": None,
+                "tag": None
+            }, {
+                "id": 11189877,
+                "type": "A",
+                "name": "@",
+                "data": "192.0.2.1",
+                "priority": None,
+                "port": None,
+                "ttl": 3600,
+                "weight": None,
+                "flags": None,
+                "tag": None
+            }],
+            "links": {},
+            "meta": {
+                "total": 4
+            }
+        }
+
+        # non-existant domain, create everything
+        resp.json.side_effect = [
+            DigitalOceanClientNotFound,  # no zone in populate
+            DigitalOceanClientNotFound,  # no domain during apply
+            domain_after_creation
+        ]
+        plan = provider.plan(self.expected)
+
+        # No root NS, no ignored, no excluded, no unsupported
+        n = len(self.expected.records) - 7
+        self.assertEquals(n, len(plan.changes))
+        self.assertEquals(n, provider.apply(plan))
+
+        provider._client._request.assert_has_calls([
+            # created the domain
+            call('POST', '/domains', data={'ip_address': '192.0.2.1',
+                                           'name': 'unit.tests'}),
+            # get all records in newly created zone
+            call('GET', '/domains/unit.tests/records', {'page': 1}),
+            # delete the initial A record
+            call('DELETE', '/domains/unit.tests/records/11189877'),
+            # created at least one of the record with expected data
+            call('POST', '/domains/unit.tests/records', data={
+                'name': '_srv._tcp',
+                'weight': 20,
+                'data': 'foo-1.unit.tests.',
+                'priority': 10,
+                'ttl': 600,
+                'type': 'SRV',
+                'port': 30
+            }),
+        ])
+        self.assertEquals(24, provider._client._request.call_count)
+
+        provider._client._request.reset_mock()
+
+        # delete 1 and update 1
+        provider._client.records = Mock(return_value=[
+            {
+                'id': 11189897,
+                'name': 'www',
+                'data': '1.2.3.4',
+                'ttl': 300,
+                'type': 'A',
+            },
+            {
+                'id': 11189898,
+                'name': 'www',
+                'data': '2.2.3.4',
+                'ttl': 300,
+                'type': 'A',
+            },
+            {
+                'id': 11189899,
+                'name': 'ttl',
+                'data': '3.2.3.4',
+                'ttl': 600,
+                'type': 'A',
+            }
+        ])
+
+        # Domain exists, we don't care about return
+        resp.json.side_effect = ['{}']
+
+        wanted = Zone('unit.tests.', [])
+        wanted.add_record(Record.new(wanted, 'ttl', {
+            'ttl': 300,
+            'type': 'A',
+            'value': '3.2.3.4'
+        }))
+
+        plan = provider.plan(wanted)
+        self.assertEquals(2, len(plan.changes))
+        self.assertEquals(2, provider.apply(plan))
+        # recreate for update, and delete for the 2 parts of the other
+        provider._client._request.assert_has_calls([
+            call('POST', '/domains/unit.tests/records', data={
+                'data': '3.2.3.4',
+                'type': 'A',
+                'name': 'ttl',
+                'ttl': 300
+            }),
+            call('DELETE', '/domains/unit.tests/records/11189899'),
+            call('DELETE', '/domains/unit.tests/records/11189897'),
+            call('DELETE', '/domains/unit.tests/records/11189898')
+        ], any_order=True)

tests/test_octodns_provider_yaml.py

@@ -49,12 +49,12 @@ class TestYamlProvider(TestCase):
 
             # We add everything
             plan = target.plan(zone)
-            self.assertEquals(14, len(filter(lambda c: isinstance(c, Create),
+            self.assertEquals(15, len(filter(lambda c: isinstance(c, Create),
                                              plan.changes)))
             self.assertFalse(isfile(yaml_file))
 
             # Now actually do it
-            self.assertEquals(14, target.apply(plan))
+            self.assertEquals(15, target.apply(plan))
             self.assertTrue(isfile(yaml_file))
 
             # There should be no changes after the round trip
@@ -64,15 +64,19 @@ class TestYamlProvider(TestCase):
 
             # A 2nd sync should still create everything
             plan = target.plan(zone)
-            self.assertEquals(14, len(filter(lambda c: isinstance(c, Create),
+            self.assertEquals(15, len(filter(lambda c: isinstance(c, Create),
                                              plan.changes)))
 
             with open(yaml_file) as fh:
                 data = safe_load(fh.read())
 
+                # '' has some of both
+                roots = sorted(data[''], key=lambda r: r['type'])
+                self.assertTrue('values' in roots[0])  # A
+                self.assertTrue('value' in roots[1])   # CAA
+                self.assertTrue('values' in roots[2])  # SSHFP
+
                 # these are stored as plural 'values'
-                for r in data['']:
-                    self.assertTrue('values' in r)
                 self.assertTrue('values' in data['mx'])
                 self.assertTrue('values' in data['naptr'])
                 self.assertTrue('values' in data['_srv._tcp'])