From ba3ad27c024cad228b5f61cbd8b429b147587645 Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Thu, 2 Nov 2017 06:50:15 -0700 Subject: [PATCH 1/9] Make PowerDnsBaseProvider's timeout configurable --- octodns/provider/powerdns.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/octodns/provider/powerdns.py b/octodns/provider/powerdns.py index 20cfe8b..4527f8e 100644 --- a/octodns/provider/powerdns.py +++ b/octodns/provider/powerdns.py @@ -18,13 +18,14 @@ class PowerDnsBaseProvider(BaseProvider): 'PTR', 'SPF', 'SSHFP', 'SRV', 'TXT')) TIMEOUT = 5 - def __init__(self, id, host, api_key, port=8081, scheme="http", *args, - **kwargs): + def __init__(self, id, host, api_key, port=8081, scheme="http", + timeout=TIMEOUT, *args, **kwargs): super(PowerDnsBaseProvider, self).__init__(id, *args, **kwargs) self.host = host self.port = port self.scheme = scheme + self.timeout = timeout sess = Session() sess.headers.update({'X-API-Key': api_key}) @@ -35,7 +36,7 @@ class PowerDnsBaseProvider(BaseProvider): url = '{}://{}:{}/api/v1/servers/localhost/{}' \ .format(self.scheme, self.host, self.port, path) - resp = self._sess.request(method, url, json=data, timeout=self.TIMEOUT) + resp = self._sess.request(method, url, json=data, timeout=self.timeout) self.log.debug('_request: status=%d', resp.status_code) resp.raise_for_status() return resp From 77d2fd1eb401adbd68410c67f24a45b30eb3f622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Fri, 3 Nov 2017 22:31:38 +0100 Subject: [PATCH 2/9] improve setuptools capabilities --- README.md | 1 + octodns/__init__.py | 25 +++++++++++++++-- requirements-dev.txt | 7 ----- requirements.txt | 23 --------------- script/bootstrap | 4 +-- setup.cfg | 67 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 46 ++---------------------------- 7 files changed, 94 insertions(+), 79 deletions(-) delete mode 100644 requirements-dev.txt delete mode 100644 requirements.txt create mode 100644 setup.cfg diff --git a/README.md b/README.md index a910b5b..ec9164f 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ $ cd dns $ virtualenv env ... $ source env/bin/activate +$ pip install -U setuptools $ pip install octodns $ mkdir config ``` diff --git a/octodns/__init__.py b/octodns/__init__.py index 2166778..aaaa2a5 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -1,6 +1,25 @@ -'OctoDNS: DNS as code - Tools for managing DNS across multiple providers' - from __future__ import absolute_import, division, print_function, \ unicode_literals +import pkg_resources +from os import path +from setuptools.config import read_configuration -__VERSION__ = '0.8.8' + +def _extract_version(package_name): + try: + return pkg_resources.get_distribution(package_name).version + except pkg_resources.DistributionNotFound: + _conf = read_configuration( + path.join( + path.dirname(path.dirname(__file__)), + 'setup.cfg' + ) + ) + return _conf['metadata']['version'] + + +__version__ = _extract_version('octodns') + + +if __name__ == "__main__": + print(__version__) diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 5cdf252..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -coverage -mock -nose -pep8 -pyflakes -requests_mock -setuptools>=36.4.0 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 80fbe1e..0000000 --- a/requirements.txt +++ /dev/null @@ -1,23 +0,0 @@ -# These are known good versions. You're free to use others and things will -# likely work, but no promises are made, especilly if you go older. -PyYaml==3.12 -azure-mgmt-dns==1.0.1 -azure-common==1.1.6 -boto3==1.4.6 -botocore==1.6.8 -dnspython==1.15.0 -docutils==0.14 -dyn==1.8.0 -futures==3.1.1 -google-cloud==0.27.0 -incf.countryutils==1.0 -ipaddress==1.0.18 -jmespath==0.9.3 -msrestazure==0.4.10 -natsort==5.0.3 -nsone==0.9.14 -ovh==0.4.7 -python-dateutil==2.6.1 -requests==2.13.0 -s3transfer==0.1.10 -six==1.10.0 diff --git a/script/bootstrap b/script/bootstrap index 1f76914..dfbb142 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -19,10 +19,10 @@ if [ ! -d "$VENV_NAME" ]; then fi . "$VENV_NAME/bin/activate" -pip install -U -r requirements.txt +pip install -e . if [ "$ENV" != "production" ]; then - pip install -U -r requirements-dev.txt + pip install -e .[dev] fi if [ ! -L ".git/hooks/pre-commit" ]; then diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..f21fdbb --- /dev/null +++ b/setup.cfg @@ -0,0 +1,67 @@ +[metadata] +name = octodns +description = "DNS as code - Tools for managing DNS across multiple providers" +long_description = file: README.md +version = 0.8.8 +author = Ross McFarland +author_email = rwmcfa1@gmail.com +url = https://github.com/github/octodns +license = MIT +keywords = dns, providers +classifiers = + License :: OSI Approved :: MIT License + Programming Language :: Python + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + +[options] +install_requires = + PyYaml==3.12 + azure-mgmt-dns==1.0.1 + azure-common==1.1.6 + boto3==1.4.6 + botocore==1.6.8 + dnspython==1.15.0 + docutils==0.14 + dyn==1.8.0 + futures==3.1.1 + google-cloud==0.27.0 + incf.countryutils==1.0 + ipaddress==1.0.18 + jmespath==0.9.3 + msrestazure==0.4.10 + natsort==5.0.3 + nsone==0.9.14 + ovh==0.4.7 + python-dateutil==2.6.1 + requests==2.13.0 + s3transfer==0.1.10 + six==1.10.0 +packages = find: +include_package_data = True + +[options.entry_points] +console_scripts = + octodns-compare = octodns.cmds.compare:main + octodns-dump = octodns.cmds.dump:main + octodns-report = octodns.cmds.report:main + octodns-sync = octodns.cmds.sync:main + octodns-validate = octodns.cmds.validate:main + +[options.packages.find] +exclude = + tests + +[options.extras_require] +dev = + coverage + mock + nose + pep8 + pyflakes + requests_mock + setuptools>=36.4.0 diff --git a/setup.py b/setup.py index f2b901d..2598061 100644 --- a/setup.py +++ b/setup.py @@ -1,47 +1,5 @@ #!/usr/bin/env python +from setuptools import setup -from os.path import dirname, join -import octodns -try: - from setuptools import find_packages, setup -except ImportError: - from distutils.core import find_packages, setup - -cmds = ( - 'compare', - 'dump', - 'report', - 'sync', - 'validate' -) -cmds_dir = join(dirname(__file__), 'octodns', 'cmds') -console_scripts = { - 'octodns-{name} = octodns.cmds.{name}:main'.format(name=name) - for name in cmds -} - -setup( - author='Ross McFarland', - author_email='rwmcfa1@gmail.com', - description=octodns.__doc__, - entry_points={ - 'console_scripts': console_scripts, - }, - install_requires=[ - 'PyYaml>=3.12', - 'dnspython>=1.15.0', - 'futures>=3.0.5', - 'incf.countryutils>=1.0', - 'ipaddress>=1.0.18', - 'natsort>=5.0.3', - 'python-dateutil>=2.6.0', - 'requests>=2.13.0' - ], - license='MIT', - long_description=open('README.md').read(), - name='octodns', - packages=find_packages(), - url='https://github.com/github/octodns', - version=octodns.__VERSION__, -) +setup() From dd692320c9f1779497df9b8cbc30c4d9ebc2337d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Sat, 4 Nov 2017 22:01:18 +0100 Subject: [PATCH 3/9] Apply review comments define 3 kinds of requirements (base, dev, test) retrieve version from __init__.py define setuptools minimal version in CI install full (base, dev, test) dependencies --- MANIFEST.in | 1 - README.md | 2 +- octodns/__init__.py | 22 +--------------------- script/bootstrap | 2 +- setup.cfg | 35 ++++++++++++++++++----------------- 5 files changed, 21 insertions(+), 41 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 3a26904..cda90ed 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,5 @@ include CONTRIBUTING.md include LICENSE include docs/* include octodns/* -include requirements*.txt include script/* include tests/* diff --git a/README.md b/README.md index ec9164f..ed1ac3b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ $ cd dns $ virtualenv env ... $ source env/bin/activate -$ pip install -U setuptools +$ pip install -U setuptools>⁼30.3.0 $ pip install octodns $ mkdir config ``` diff --git a/octodns/__init__.py b/octodns/__init__.py index aaaa2a5..05a5e84 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -1,25 +1,5 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import pkg_resources -from os import path -from setuptools.config import read_configuration -def _extract_version(package_name): - try: - return pkg_resources.get_distribution(package_name).version - except pkg_resources.DistributionNotFound: - _conf = read_configuration( - path.join( - path.dirname(path.dirname(__file__)), - 'setup.cfg' - ) - ) - return _conf['metadata']['version'] - - -__version__ = _extract_version('octodns') - - -if __name__ == "__main__": - print(__version__) +__version__ = '0.8.8' diff --git a/script/bootstrap b/script/bootstrap index dfbb142..7f4a5a8 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -22,7 +22,7 @@ fi pip install -e . if [ "$ENV" != "production" ]; then - pip install -e .[dev] + pip install -e .[dev,test] fi if [ ! -L ".git/hooks/pre-commit" ]; then diff --git a/setup.cfg b/setup.cfg index f21fdbb..70baaf6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ name = octodns description = "DNS as code - Tools for managing DNS across multiple providers" long_description = file: README.md -version = 0.8.8 +version = attr: octodns.__version__ author = Ross McFarland author_email = rwmcfa1@gmail.com url = https://github.com/github/octodns @@ -20,27 +20,14 @@ classifiers = [options] install_requires = - PyYaml==3.12 - azure-mgmt-dns==1.0.1 - azure-common==1.1.6 - boto3==1.4.6 - botocore==1.6.8 - dnspython==1.15.0 - docutils==0.14 - dyn==1.8.0 + PyYaml>=3.12 + dnspython>=1.15.0 futures==3.1.1 - google-cloud==0.27.0 incf.countryutils==1.0 ipaddress==1.0.18 - jmespath==0.9.3 - msrestazure==0.4.10 natsort==5.0.3 - nsone==0.9.14 - ovh==0.4.7 python-dateutil==2.6.1 requests==2.13.0 - s3transfer==0.1.10 - six==1.10.0 packages = find: include_package_data = True @@ -57,7 +44,21 @@ exclude = tests [options.extras_require] -dev = +dev = + azure-mgmt-dns==1.0.1 + azure-common==1.1.6 + boto3==1.4.6 + botocore==1.6.8 + docutils==0.14 + dyn==1.8.0 + google-cloud==0.27.0 + jmespath==0.9.3 + msrestazure==0.4.10 + nsone==0.9.14 + ovh==0.4.7 + s3transfer==0.1.10 + six==1.10.0 +test = coverage mock nose From 2ee1a41a78ba948fb8e4e07b4991a8c0efb92cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Sun, 5 Nov 2017 12:37:06 +0100 Subject: [PATCH 4/9] Remove setuptools minimal version adding minimal version only for contributors --- CONTRIBUTING.md | 4 ++++ README.md | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9a5709a..36337eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,6 +38,10 @@ Here are a few things you can do that will increase the likelihood of your pull - Write a [good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). +## Development prerequisites + +- setuptools >= 30.3.0 + ## License note We can only accept contributions that are compatible with the MIT license. diff --git a/README.md b/README.md index ed1ac3b..a910b5b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ $ cd dns $ virtualenv env ... $ source env/bin/activate -$ pip install -U setuptools>⁼30.3.0 $ pip install octodns $ mkdir config ``` From 7fa999953fc3229f95053d0c36a6ee14f1251c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Beraud?= Date: Sun, 5 Nov 2017 13:38:31 +0100 Subject: [PATCH 5/9] reset changes on __init__ fix error on version --- octodns/__init__.py | 5 +++-- setup.cfg | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/octodns/__init__.py b/octodns/__init__.py index 05a5e84..2166778 100644 --- a/octodns/__init__.py +++ b/octodns/__init__.py @@ -1,5 +1,6 @@ +'OctoDNS: DNS as code - Tools for managing DNS across multiple providers' + from __future__ import absolute_import, division, print_function, \ unicode_literals - -__version__ = '0.8.8' +__VERSION__ = '0.8.8' diff --git a/setup.cfg b/setup.cfg index 70baaf6..54e9014 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ name = octodns description = "DNS as code - Tools for managing DNS across multiple providers" long_description = file: README.md -version = attr: octodns.__version__ +version = attr: octodns.__VERSION__ author = Ross McFarland author_email = rwmcfa1@gmail.com url = https://github.com/github/octodns From 454f7f8c8fe9757a23a33ca52ced0ebc32a7cdbe Mon Sep 17 00:00:00 2001 From: Ross McFarland Date: Wed, 8 Nov 2017 06:26:18 -0800 Subject: [PATCH 6/9] Add formal CAA support to YamlProvider --- octodns/provider/yaml.py | 4 ++-- tests/test_octodns_manager.py | 14 +++++++------- tests/test_octodns_provider_yaml.py | 14 +++++++++----- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index fe1a406..752e793 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -31,8 +31,8 @@ class YamlProvider(BaseProvider): enforce_order: True ''' SUPPORTS_GEO = True - SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CNAME', 'MX', 'NAPTR', 'NS', 'PTR', - 'SSHFP', 'SPF', 'SRV', 'TXT')) + SUPPORTS = set(('A', 'AAAA', 'ALIAS', 'CAA', 'CNAME', 'MX', 'NAPTR', 'NS', + 'PTR', 'SSHFP', 'SPF', 'SRV', 'TXT')) def __init__(self, id, directory, default_ttl=3600, enforce_order=True, *args, **kwargs): diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 36b1f4c..4db2103 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -102,12 +102,12 @@ class TestManager(TestCase): environ['YAML_TMP_DIR'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False) - self.assertEquals(20, tc) + self.assertEquals(21, tc) # try with just one of the zones tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, eligible_zones=['unit.tests.']) - self.assertEquals(14, tc) + self.assertEquals(15, tc) # the subzone, with 2 targets tc = Manager(get_config_filename('simple.yaml')) \ @@ -122,18 +122,18 @@ class TestManager(TestCase): # Again with force tc = Manager(get_config_filename('simple.yaml')) \ .sync(dry_run=False, force=True) - self.assertEquals(20, tc) + self.assertEquals(21, tc) # Again with max_workers = 1 tc = Manager(get_config_filename('simple.yaml'), max_workers=1) \ .sync(dry_run=False, force=True) - self.assertEquals(20, tc) + self.assertEquals(21, tc) # Include meta tc = Manager(get_config_filename('simple.yaml'), max_workers=1, include_meta=True) \ .sync(dry_run=False, force=True) - self.assertEquals(24, tc) + self.assertEquals(25, tc) def test_eligible_targets(self): with TemporaryDirectory() as tmpdir: @@ -159,13 +159,13 @@ class TestManager(TestCase): fh.write('---\n{}') changes = manager.compare(['in'], ['dump'], 'unit.tests.') - self.assertEquals(14, len(changes)) + self.assertEquals(15, len(changes)) # Compound sources with varying support changes = manager.compare(['in', 'nosshfp'], ['dump'], 'unit.tests.') - self.assertEquals(13, len(changes)) + self.assertEquals(14, len(changes)) with self.assertRaises(Exception) as ctx: manager.compare(['nope'], ['dump'], 'unit.tests.') diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index a199355..46363ed 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -49,12 +49,12 @@ class TestYamlProvider(TestCase): # We add everything plan = target.plan(zone) - self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), + self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), plan.changes))) self.assertFalse(isfile(yaml_file)) # Now actually do it - self.assertEquals(14, target.apply(plan)) + self.assertEquals(15, target.apply(plan)) self.assertTrue(isfile(yaml_file)) # There should be no changes after the round trip @@ -64,15 +64,19 @@ class TestYamlProvider(TestCase): # A 2nd sync should still create everything plan = target.plan(zone) - self.assertEquals(14, len(filter(lambda c: isinstance(c, Create), + self.assertEquals(15, len(filter(lambda c: isinstance(c, Create), plan.changes))) with open(yaml_file) as fh: data = safe_load(fh.read()) + # '' has some of both + roots = sorted(data[''], key=lambda r: r['type']) + self.assertTrue('values' in roots[0]) # A + self.assertTrue('value' in roots[1]) # CAA + self.assertTrue('values' in roots[2]) # SSHFP + # these are stored as plural 'values' - for r in data['']: - self.assertTrue('values' in r) self.assertTrue('values' in data['mx']) self.assertTrue('values' in data['naptr']) self.assertTrue('values' in data['_srv._tcp']) From b4ead495f5069852e6925e64603caff649da279c Mon Sep 17 00:00:00 2001 From: Tim Hughes Date: Wed, 8 Nov 2017 14:41:48 +0000 Subject: [PATCH 7/9] adds an example of how to setup geodns to the docs --- docs/records.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/records.md b/docs/records.md index d991311..82d9a37 100644 --- a/docs/records.md +++ b/docs/records.md @@ -26,6 +26,55 @@ GeoDNS is currently supported for `A` and `AAAA` records on the Dyn (via Traffic Configuring GeoDNS is complex and the details of the functionality vary widely from provider to provider. OctoDNS has an opinionated view of how GeoDNS should be set up and does its best to map that to each provider's offering in a way that will result in similar behavior. It may not fit your needs or use cases, in which case please open an issue for discussion. We expect this functionality to grow and evolve over time as it's more widely used. +The following is an example of GeoDNS with three entries NA-US-CA, NA-US-NY, OC-AU. Octodns creates another one labeled 'default' with the details for the actual A record, This default record is the failover record if the monitoring check fails. + +```yaml +--- +? '' +: type: TXT + value: v=spf1 -all +test: + geo: + NA-US-NY: + - 111.111.111.1 + NA-US-CA: + - 111.111.111.2 + OC-AU: + - 111.111.111.3 + EU: + - 111.111.111.4 + ttl: 300 + type: A + value: 111.111.111.5 +``` + + +The geo labels breakdown based on: + +1. + - 'AF': 14, # Continental Africa + - 'AN': 17, # Continental Antartica + - 'AS': 15, # Contentinal Asia + - 'EU': 13, # Contentinal Europe + - 'NA': 11, # Continental North America + - 'OC': 16, # Contentinal Austrailia/Oceania + - 'SA': 12, # Continental South America + +2. ISO Country Code https://en.wikipedia.org/wiki/ISO_3166-2 + +3. ISO Country Code Subdevision as per https://en.wikipedia.org/wiki/ISO_3166-2:US (change the code at the end for the country you are subdividing) * these may not always be supported depending on the provider. + +So the example is saying: + +- North America - United States - New York: gets served an "A" record of 111.111.111.1 +- North America - United States - California: gets served an "A" record of 111.111.111.2 +- Oceania - Australia: Gets served an "A" record of 111.111.111.3 +- Europe: gets an "A" record of 111.111.111.4 +- Everyone else gets an "A" record of 111.111.111.5 + + +Octodns will automatically set up a monitor and check for **https:///_dns** and check for a 200 response. + ## Config (`YamlProvider`) OctoDNS records and `YamlProvider`'s schema is essentially a 1:1 match. Properties on the objects will match keys in the config. From feec4a68215b5bc8f85b9e1d3f491ada08a84cae Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Mon, 6 Nov 2017 22:32:46 -0800 Subject: [PATCH 8/9] Add DigitalOcean provider --- README.md | 1 + octodns/provider/digitalocean.py | 336 ++++++++++++++++++++ tests/fixtures/digitalocean-page-1.json | 177 +++++++++++ tests/fixtures/digitalocean-page-2.json | 89 ++++++ tests/test_octodns_provider_digitalocean.py | 241 ++++++++++++++ 5 files changed, 844 insertions(+) create mode 100644 octodns/provider/digitalocean.py create mode 100644 tests/fixtures/digitalocean-page-1.json create mode 100644 tests/fixtures/digitalocean-page-2.json create mode 100644 tests/test_octodns_provider_digitalocean.py diff --git a/README.md b/README.md index a910b5b..88223e6 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,7 @@ The above command pulled the existing data out of Route53 and placed the results |--|--|--|--| | [AzureProvider](/octodns/provider/azuredns.py) | A, AAAA, CNAME, MX, NS, PTR, SRV, TXT | No | | | [CloudflareProvider](/octodns/provider/cloudflare.py) | A, AAAA, CAA, CNAME, MX, NS, SPF, TXT | No | CAA tags restricted | +| [DigitalOceanProvider](/octodns/provider/digitalocean.py) | A, AAAA, CAA, CNAME, MX, NS, TXT, SRV | No | CAA tags restricted | | [DnsimpleProvider](/octodns/provider/dnsimple.py) | All | No | CAA tags restricted | | [DynProvider](/octodns/provider/dyn.py) | All | Yes | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | diff --git a/octodns/provider/digitalocean.py b/octodns/provider/digitalocean.py new file mode 100644 index 0000000..55fa10b --- /dev/null +++ b/octodns/provider/digitalocean.py @@ -0,0 +1,336 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from collections import defaultdict +from requests import Session +import logging + +from ..record import Record +from .base import BaseProvider + + +class DigitalOceanClientException(Exception): + pass + + +class DigitalOceanClientNotFound(DigitalOceanClientException): + + def __init__(self): + super(DigitalOceanClientNotFound, self).__init__('Not Found') + + +class DigitalOceanClientUnauthorized(DigitalOceanClientException): + + def __init__(self): + super(DigitalOceanClientUnauthorized, self).__init__('Unauthorized') + + +class DigitalOceanClient(object): + BASE = 'https://api.digitalocean.com/v2' + + def __init__(self, token): + sess = Session() + sess.headers.update({'Authorization': 'Bearer {}'.format(token)}) + self._sess = sess + + def _request(self, method, path, params=None, data=None): + url = '{}{}'.format(self.BASE, path) + resp = self._sess.request(method, url, params=params, json=data) + if resp.status_code == 401: + raise DigitalOceanClientUnauthorized() + if resp.status_code == 404: + raise DigitalOceanClientNotFound() + resp.raise_for_status() + return resp + + def domain(self, name): + path = '/domains/{}'.format(name) + return self._request('GET', path).json() + + def domain_create(self, name): + # Digitalocean requires an IP on zone creation + self._request('POST', '/domains', data={'name': name, + 'ip_address': '192.0.2.1'}) + + # After the zone is created, immeadiately delete the record + records = self.records(name) + for record in records: + if record['name'] == '' and record['type'] == 'A': + self.record_delete(name, record['id']) + + def records(self, zone_name): + path = '/domains/{}/records'.format(zone_name) + ret = [] + + page = 1 + while True: + data = self._request('GET', path, {'page': page}).json() + + ret += data['domain_records'] + links = data['links'] + + # https://developers.digitalocean.com/documentation/v2/#links + # pages exists if there is more than 1 page + # last doesn't exist if you're on the last page + try: + links['pages']['last'] + page += 1 + except KeyError: + break + + # change any apex record to empty string to match other provider output + for record in ret: + if record['name'] == '@': + record['name'] = '' + + return ret + + def record_create(self, zone_name, params): + path = '/domains/{}/records'.format(zone_name) + # change empty string to @, DigitalOcean uses @ for apex record names + if params['name'] == '': + params['name'] = '@' + self._request('POST', path, data=params) + + def record_delete(self, zone_name, record_id): + path = '/domains/{}/records/{}'.format(zone_name, record_id) + self._request('DELETE', path) + + +class DigitalOceanProvider(BaseProvider): + ''' + DigitalOcean DNS provider using API v2 + + digitalocean: + class: octodns.provider.digitalocean.DigitalOceanProvider + # Your DigitalOcean API token (required) + token: foo + ''' + SUPPORTS_GEO = False + SUPPORTS = set(('A', 'AAAA', 'CAA', 'CNAME', 'MX', 'NS', 'TXT', 'SRV')) + + def __init__(self, id, token, *args, **kwargs): + self.log = logging.getLogger('DigitalOceanProvider[{}]'.format(id)) + self.log.debug('__init__: id=%s, token=***', id) + super(DigitalOceanProvider, self).__init__(id, *args, **kwargs) + self._client = DigitalOceanClient(token) + + self._zone_records = {} + + def _data_for_multiple(self, _type, records): + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': [r['data'] for r in records] + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_CAA(self, _type, records): + values = [] + for record in records: + values.append({ + 'flags': record['flags'], + 'tag': record['tag'], + 'value': record['data'], + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_CNAME(self, _type, records): + record = records[0] + return { + 'ttl': record['ttl'], + 'type': _type, + 'value': '{}.'.format(record['data']) + } + + def _data_for_MX(self, _type, records): + values = [] + for record in records: + values.append({ + 'preference': record['priority'], + 'exchange': '{}.'.format(record['data']) + }) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def _data_for_NS(self, _type, records): + values = [] + for record in records: + data = '{}.'.format(record['data']) + values.append(data) + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values, + } + + def _data_for_SRV(self, _type, records): + values = [] + for record in records: + values.append({ + 'port': record['port'], + 'priority': record['priority'], + 'target': '{}.'.format(record['data']), + 'weight': record['weight'] + }) + return { + 'type': _type, + 'ttl': records[0]['ttl'], + 'values': values + } + + def _data_for_TXT(self, _type, records): + values = [value['data'].replace(';', '\;') for value in records] + return { + 'ttl': records[0]['ttl'], + 'type': _type, + 'values': values + } + + def zone_records(self, zone): + if zone.name not in self._zone_records: + try: + self._zone_records[zone.name] = \ + self._client.records(zone.name[:-1]) + except DigitalOceanClientNotFound: + return [] + + return self._zone_records[zone.name] + + def populate(self, zone, target=False, lenient=False): + self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name, + target, lenient) + + values = defaultdict(lambda: defaultdict(list)) + for record in self.zone_records(zone): + _type = record['type'] + values[record['name']][record['type']].append(record) + + before = len(zone.records) + for name, types in values.items(): + for _type, records in types.items(): + data_for = getattr(self, '_data_for_{}'.format(_type)) + record = Record.new(zone, name, data_for(_type, records), + source=self, lenient=lenient) + zone.add_record(record) + + self.log.info('populate: found %s records', + len(zone.records) - before) + + def _params_for_multiple(self, record): + for value in record.values: + yield { + 'data': value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + + def _params_for_CAA(self, record): + for value in record.values: + yield { + 'data': '{}.'.format(value.value), + 'flags': value.flags, + 'name': record.name, + 'tag': value.tag, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_single(self, record): + yield { + 'data': record.value, + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + _params_for_CNAME = _params_for_single + + def _params_for_MX(self, record): + for value in record.values: + yield { + 'data': value.exchange, + 'name': record.name, + 'priority': value.preference, + 'ttl': record.ttl, + 'type': record._type + } + + def _params_for_SRV(self, record): + for value in record.values: + yield { + 'data': value.target, + 'name': record.name, + 'port': value.port, + 'priority': value.priority, + 'ttl': record.ttl, + 'type': record._type, + 'weight': value.weight + } + + def _params_for_TXT(self, record): + # DigitalOcean doesn't want things escaped in values so we + # have to strip them here and add them when going the other way + for value in record.values: + yield { + 'data': value.replace('\;', ';'), + 'name': record.name, + 'ttl': record.ttl, + 'type': record._type + } + + def _apply_Create(self, change): + new = change.new + params_for = getattr(self, '_params_for_{}'.format(new._type)) + for params in params_for(new): + self._client.record_create(new.zone.name[:-1], params) + + def _apply_Update(self, change): + self._apply_Delete(change) + self._apply_Create(change) + + def _apply_Delete(self, change): + existing = change.existing + zone = existing.zone + for record in self.zone_records(zone): + if existing.name == record['name'] and \ + existing._type == record['type']: + self._client.record_delete(zone.name[:-1], record['id']) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + self.log.debug('_apply: zone=%s, len(changes)=%d', desired.name, + len(changes)) + + domain_name = desired.name[:-1] + try: + self._client.domain(domain_name) + except DigitalOceanClientNotFound: + self.log.debug('_apply: no matching zone, creating domain') + self._client.domain_create(domain_name) + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, '_apply_{}'.format(class_name))(change) + + # Clear out the cache if any + self._zone_records.pop(desired.name, None) diff --git a/tests/fixtures/digitalocean-page-1.json b/tests/fixtures/digitalocean-page-1.json new file mode 100644 index 0000000..db231ba --- /dev/null +++ b/tests/fixtures/digitalocean-page-1.json @@ -0,0 +1,177 @@ +{ + "domain_records": [{ + "id": 11189874, + "type": "NS", + "name": "@", + "data": "ns1.digitalocean.com", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189875, + "type": "NS", + "name": "@", + "data": "ns2.digitalocean.com", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189876, + "type": "NS", + "name": "@", + "data": "ns3.digitalocean.com", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189877, + "type": "NS", + "name": "under", + "data": "ns1.unit.tests", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189878, + "type": "NS", + "name": "under", + "data": "ns2.unit.tests", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189879, + "type": "SRV", + "name": "_srv._tcp", + "data": "foo-1.unit.tests", + "priority": 10, + "port": 30, + "ttl": 600, + "weight": 20, + "flags": null, + "tag": null + }, { + "id": 11189880, + "type": "SRV", + "name": "_srv._tcp", + "data": "foo-2.unit.tests", + "priority": 12, + "port": 30, + "ttl": 600, + "weight": 20, + "flags": null, + "tag": null + }, { + "id": 11189881, + "type": "TXT", + "name": "txt", + "data": "Bah bah black sheep", + "priority": null, + "port": null, + "ttl": 600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189882, + "type": "TXT", + "name": "txt", + "data": "have you any wool.", + "priority": null, + "port": null, + "ttl": 600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189883, + "type": "A", + "name": "@", + "data": "1.2.3.4", + "priority": null, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189884, + "type": "A", + "name": "@", + "data": "1.2.3.5", + "priority": null, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189885, + "type": "A", + "name": "www", + "data": "2.2.3.6", + "priority": null, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189886, + "type": "MX", + "name": "mx", + "data": "smtp-4.unit.tests", + "priority": 10, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189887, + "type": "MX", + "name": "mx", + "data": "smtp-2.unit.tests", + "priority": 20, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189888, + "type": "MX", + "name": "mx", + "data": "smtp-3.unit.tests", + "priority": 30, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }], + "links": { + "pages": { + "last": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=2", + "next": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=2" + } + }, + "meta": { + "total": 21 + } +} \ No newline at end of file diff --git a/tests/fixtures/digitalocean-page-2.json b/tests/fixtures/digitalocean-page-2.json new file mode 100644 index 0000000..8b989ae --- /dev/null +++ b/tests/fixtures/digitalocean-page-2.json @@ -0,0 +1,89 @@ +{ + "domain_records": [{ + "id": 11189889, + "type": "MX", + "name": "mx", + "data": "smtp-1.unit.tests", + "priority": 40, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189890, + "type": "AAAA", + "name": "aaaa", + "data": "2601:644:500:e210:62f8:1dff:feb8:947a", + "priority": null, + "port": null, + "ttl": 600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189891, + "type": "CNAME", + "name": "cname", + "data": "unit.tests", + "priority": null, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189892, + "type": "A", + "name": "www.sub", + "data": "2.2.3.6", + "priority": null, + "port": null, + "ttl": 300, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189893, + "type": "TXT", + "name": "txt", + "data": "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs", + "priority": null, + "port": null, + "ttl": 600, + "weight": null, + "flags": null, + "tag": null + }, { + "id": 11189894, + "type": "CAA", + "name": "@", + "data": "ca.unit.tests", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": 0, + "tag": "issue" + }, { + "id": 11189895, + "type": "CNAME", + "name": "included", + "data": "unit.tests", + "priority": null, + "port": null, + "ttl": 3600, + "weight": null, + "flags": null, + "tag": null + }], + "links": { + "pages": { + "first": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=1", + "prev": "https://api.digitalocean.com/v2/domains/unit.tests/records?page=1" + } + }, + "meta": { + "total": 21 + } +} \ No newline at end of file diff --git a/tests/test_octodns_provider_digitalocean.py b/tests/test_octodns_provider_digitalocean.py new file mode 100644 index 0000000..5fb47c5 --- /dev/null +++ b/tests/test_octodns_provider_digitalocean.py @@ -0,0 +1,241 @@ +# +# +# + + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from mock import Mock, call +from os.path import dirname, join +from requests import HTTPError +from requests_mock import ANY, mock as requests_mock +from unittest import TestCase + +from octodns.record import Record +from octodns.provider.digitalocean import DigitalOceanClientNotFound, \ + DigitalOceanProvider +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestDigitalOceanProvider(TestCase): + expected = Zone('unit.tests.', []) + source = YamlProvider('test', join(dirname(__file__), 'config')) + source.populate(expected) + + # Our test suite differs a bit, add our NS and remove the simple one + expected.add_record(Record.new(expected, 'under', { + 'ttl': 3600, + 'type': 'NS', + 'values': [ + 'ns1.unit.tests.', + 'ns2.unit.tests.', + ] + })) + for record in list(expected.records): + if record.name == 'sub' and record._type == 'NS': + expected._remove_record(record) + break + + def test_populate(self): + provider = DigitalOceanProvider('test', 'token') + + # Bad auth + with requests_mock() as mock: + mock.get(ANY, status_code=401, + text='{"id":"unauthorized",' + '"message":"Unable to authenticate you."}') + + with self.assertRaises(Exception) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals('Unauthorized', ctx.exception.message) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=502, text='Things caught fire') + + with self.assertRaises(HTTPError) as ctx: + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(502, ctx.exception.response.status_code) + + # Non-existant zone doesn't populate anything + with requests_mock() as mock: + mock.get(ANY, status_code=404, + text='{"id":"not_found","message":"The resource you ' + 'were accessing could not be found."}') + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(set(), zone.records) + + # No diffs == no changes + with requests_mock() as mock: + base = 'https://api.digitalocean.com/v2/domains/unit.tests/' \ + 'records?page=' + with open('tests/fixtures/digitalocean-page-1.json') as fh: + mock.get('{}{}'.format(base, 1), text=fh.read()) + with open('tests/fixtures/digitalocean-page-2.json') as fh: + mock.get('{}{}'.format(base, 2), text=fh.read()) + + zone = Zone('unit.tests.', []) + provider.populate(zone) + self.assertEquals(12, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 2nd populate makes no network calls/all from cache + again = Zone('unit.tests.', []) + provider.populate(again) + self.assertEquals(12, len(again.records)) + + # bust the cache + del provider._zone_records[zone.name] + + def test_apply(self): + provider = DigitalOceanProvider('test', 'token') + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + domain_after_creation = { + "domain_records": [{ + "id": 11189874, + "type": "NS", + "name": "@", + "data": "ns1.digitalocean.com", + "priority": None, + "port": None, + "ttl": 3600, + "weight": None, + "flags": None, + "tag": None + }, { + "id": 11189875, + "type": "NS", + "name": "@", + "data": "ns2.digitalocean.com", + "priority": None, + "port": None, + "ttl": 3600, + "weight": None, + "flags": None, + "tag": None + }, { + "id": 11189876, + "type": "NS", + "name": "@", + "data": "ns3.digitalocean.com", + "priority": None, + "port": None, + "ttl": 3600, + "weight": None, + "flags": None, + "tag": None + }, { + "id": 11189877, + "type": "A", + "name": "@", + "data": "192.0.2.1", + "priority": None, + "port": None, + "ttl": 3600, + "weight": None, + "flags": None, + "tag": None + }], + "links": {}, + "meta": { + "total": 4 + } + } + + # non-existant domain, create everything + resp.json.side_effect = [ + DigitalOceanClientNotFound, # no zone in populate + DigitalOceanClientNotFound, # no domain during apply + domain_after_creation + ] + plan = provider.plan(self.expected) + + # No root NS, no ignored, no excluded, no unsupported + n = len(self.expected.records) - 7 + self.assertEquals(n, len(plan.changes)) + self.assertEquals(n, provider.apply(plan)) + + provider._client._request.assert_has_calls([ + # created the domain + call('POST', '/domains', data={'ip_address': '192.0.2.1', + 'name': 'unit.tests'}), + # get all records in newly created zone + call('GET', '/domains/unit.tests/records', {'page': 1}), + # delete the initial A record + call('DELETE', '/domains/unit.tests/records/11189877'), + # created at least one of the record with expected data + call('POST', '/domains/unit.tests/records', data={ + 'name': '_srv._tcp', + 'weight': 20, + 'data': 'foo-1.unit.tests.', + 'priority': 10, + 'ttl': 600, + 'type': 'SRV', + 'port': 30 + }), + ]) + self.assertEquals(24, provider._client._request.call_count) + + provider._client._request.reset_mock() + + # delete 1 and update 1 + provider._client.records = Mock(return_value=[ + { + 'id': 11189897, + 'name': 'www', + 'data': '1.2.3.4', + 'ttl': 300, + 'type': 'A', + }, + { + 'id': 11189898, + 'name': 'www', + 'data': '2.2.3.4', + 'ttl': 300, + 'type': 'A', + }, + { + 'id': 11189899, + 'name': 'ttl', + 'data': '3.2.3.4', + 'ttl': 600, + 'type': 'A', + } + ]) + + # Domain exists, we don't care about return + resp.json.side_effect = ['{}'] + + wanted = Zone('unit.tests.', []) + wanted.add_record(Record.new(wanted, 'ttl', { + 'ttl': 300, + 'type': 'A', + 'value': '3.2.3.4' + })) + + plan = provider.plan(wanted) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + # recreate for update, and delete for the 2 parts of the other + provider._client._request.assert_has_calls([ + call('POST', '/domains/unit.tests/records', data={ + 'data': '3.2.3.4', + 'type': 'A', + 'name': 'ttl', + 'ttl': 300 + }), + call('DELETE', '/domains/unit.tests/records/11189899'), + call('DELETE', '/domains/unit.tests/records/11189897'), + call('DELETE', '/domains/unit.tests/records/11189898') + ], any_order=True) From 75cfc4fb76a43c38db55cd9eeb511aa7e97dcb2e Mon Sep 17 00:00:00 2001 From: Adam Smith Date: Mon, 13 Nov 2017 00:19:05 -0800 Subject: [PATCH 9/9] remove default config file for octodns-validate --- octodns/cmds/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/cmds/validate.py b/octodns/cmds/validate.py index 85c3018..6711ec9 100755 --- a/octodns/cmds/validate.py +++ b/octodns/cmds/validate.py @@ -15,7 +15,7 @@ from octodns.manager import Manager def main(): parser = ArgumentParser(description=__doc__.split('\n')[1]) - parser.add_argument('--config-file', default='./config/production.yaml', + parser.add_argument('--config-file', required=True, help='The Manager configuration file to use') args = parser.parse_args(WARN)