Constellix customized healthcheck

docs/dynamic_records.md

@@ -151,3 +151,26 @@ Support matrix:
         measure_latency: false
         request_interval: 30
 ```
+
+#### Constellix Health Check Options
+
+| Key  | Description | Default |
+|--|--|--|
+| sonar_interval | Sonar check interval [FIVESECONDS|THIRTYSECONDS|ONEMINUTE|TWOMINUTES|THREEMINUTES|FOURMINUTES|FIVEMINUTES|TENMINUTES|THIRTYMINUTES|HALFDAY|DAY] | ONEMINUTE |
+| sonar_port | Sonar check port | 80 |
+| sonar_regions | Sonar check regions for a check. WORLD or a list of [ASIAPAC|EUROPE|NACENTRAL|NAEAST|NAWEST|OCEANIA|SOUTHAMERICA] | WORLD |
+| sonar_type | Sonar check type [TCP|HTTP] | TCP |
+
+```yaml
+
+---
+  octodns:
+    constellix:
+      healthcheck:
+        sonar_interval: DAY
+        sonar_port: 80
+        sonar_regions:
+        - ASIAPAC
+        - EUROPE
+        sonar_type: TCP
+```

octodns/provider/constellix.py

@@ -7,7 +7,7 @@ from __future__ import absolute_import, division, print_function, \
 
 from collections import defaultdict
 from requests import Session
-from base64 import b64encode
+from base64 import b64encode, standard_b64encode
 from pycountry_convert import country_alpha2_to_continent_code
 import hashlib
 import hmac
@@ -191,6 +191,7 @@ class ConstellixClient(object):
         for pool in pools:
             if pool['id'] == pool_id:
                 return pool
+        return None
 
     def pool_create(self, data):
         path = f'/pools/{data.get("type")}'
@@ -241,6 +242,7 @@ class ConstellixClient(object):
         for geofilter in geofilters:
             if geofilter['id'] == geofilter_id:
                 return geofilter
+        return None
 
     def geofilter_create(self, data):
         path = '/geoFilters'
@@ -270,6 +272,149 @@ class ConstellixClient(object):
             self._geofilters.pop(geofilter_id, None)
 
 
+class SonarClientException(ProviderException):
+    pass
+
+
+class SonarClientBadRequest(SonarClientException):
+
+    def __init__(self, resp):
+        errors = resp.text
+        super(SonarClientBadRequest, self).__init__(f'\n  - {errors}')
+
+
+class SonarClientUnauthorized(SonarClientException):
+
+    def __init__(self):
+        super(SonarClientUnauthorized, self).__init__('Unauthorized')
+
+
+class SonarClientNotFound(SonarClientException):
+
+    def __init__(self):
+        super(SonarClientNotFound, self).__init__('Not Found')
+
+
+class SonarClient(object):
+    BASE = 'https://api.sonar.constellix.com/rest/api'
+
+    def __init__(self, api_key, secret_key, ratelimit_delay=0.0):
+        self.api_key = api_key
+        self.secret_key = secret_key
+        self.ratelimit_delay = ratelimit_delay
+        self._sess = Session()
+        self._agents = None
+        self._checks = {'tcp': None, 'http': None}
+
+    def _current_time(self):
+        return str(int(time.time() * 1000))
+
+    def _hmac_hash(self, now):
+        digester = hmac.new(
+            bytes(self.secret_key, "UTF-8"),
+            bytes(now, "UTF-8"),
+            hashlib.sha1)
+        signature = digester.digest()
+        hmac_text = str(standard_b64encode(signature), "UTF-8")
+        return hmac_text
+
+    def _request(self, method, path, params=None, data=None):
+        now = self._current_time()
+        hmac_text = self._hmac_hash(now)
+
+        headers = {
+            'x-cns-security-token': "{}:{}:{}".format(
+                self.api_key,
+                hmac_text,
+                now),
+            'Content-Type': "application/json"
+        }
+
+        url = f'{self.BASE}{path}'
+        resp = self._sess.request(method, url, headers=headers,
+                                  params=params, json=data)
+        if resp.status_code == 400:
+            raise SonarClientBadRequest(resp)
+        if resp.status_code == 401:
+            raise SonarClientUnauthorized()
+        if resp.status_code == 404:
+            raise SonarClientNotFound()
+        resp.raise_for_status()
+        time.sleep(self.ratelimit_delay)
+        return resp
+
+    @property
+    def agents(self):
+        if self._agents is None:
+            agents = []
+
+            resp = self._request('GET', '/system/sites').json()
+            agents += resp
+
+            self._agents = {f'{a["name"]}.': a for a in agents}
+
+        return self._agents
+
+    def agents_for_regions(self, regions):
+        if regions[0] == "WORLD":
+            res_agents = []
+            for agent in self.agents.values():
+                res_agents.append(agent['id'])
+            return res_agents
+
+        res_agents = []
+        for agent in self.agents.values():
+            if agent["region"] in regions:
+                res_agents.append(agent['id'])
+        return res_agents
+
+    def parse_uri_id(self, url):
+        r = str(url).rfind("/")
+        res = str(url)[r + 1:]
+        return res
+
+    def checks(self, check_type):
+        if self._checks[check_type] is None:
+            self._checks[check_type] = {}
+            path = f'/{check_type}'
+            response = self._request('GET', path).json()
+            for check in response:
+                self._checks[check_type][check['id']] = check
+        return self._checks[check_type].values()
+
+    def check(self, check_type, check_name):
+        checks = self.checks(check_type)
+        for check in checks:
+            if check['name'] == check_name:
+                return check
+        return None
+
+    def check_create(self, check_type, data):
+        path = f'/{check_type}'
+        response = self._request('POST', path, data=data)
+        # Parse check ID from Location response header
+        id = self.parse_uri_id(response.headers["Location"])
+        # Get check details
+        path = f'/{check_type}/{id}'
+        response = self._request('GET', path, data=data).json()
+
+        # Update our cache
+        self._checks[check_type]['id'] = response
+        return response
+
+    def check_delete(self, check_id):
+        # first get check type
+        path = f'/check/type/{check_id}'
+        response = self._request('GET', path).json()
+        check_type = response['type'].lower()
+
+        path = f'/{check_type}/{check_id}'
+        self._request('DELETE', path)
+
+        # Update our cache
+        self._checks[check_type].pop(check_id, None)
+
+
 class ConstellixProvider(BaseProvider):
     '''
     Constellix DNS provider
@@ -295,6 +440,7 @@ class ConstellixProvider(BaseProvider):
         self.log.debug('__init__: id=%s, api_key=***, secret_key=***', id)
         super(ConstellixProvider, self).__init__(id, *args, **kwargs)
         self._client = ConstellixClient(api_key, secret_key, ratelimit_delay)
+        self._sonar = SonarClient(api_key, secret_key, ratelimit_delay)
         self._zone_records = {}
 
     def _data_for_multiple(self, _type, records):
@@ -511,6 +657,27 @@ class ConstellixProvider(BaseProvider):
                       len(zone.records) - before, exists)
         return exists
 
+    def _healthcheck_config(self, record):
+        sonar_healthcheck = record._octodns.get('constellix', {}) \
+            .get('healthcheck', None)
+
+        if sonar_healthcheck is None:
+            return None
+
+        healthcheck = {}
+        healthcheck["sonar_port"] = sonar_healthcheck.get('sonar_port', 80)
+        healthcheck["sonar_type"] = sonar_healthcheck.get('sonar_type', "TCP")
+        healthcheck["sonar_regions"] = sonar_healthcheck.get(
+            'sonar_regions',
+            ["WORLD"]
+        )
+        healthcheck["sonar_interval"] = sonar_healthcheck.get(
+            'sonar_interval',
+            "ONEMINUTE"
+        )
+
+        return healthcheck
+
     def _params_for_multiple(self, record):
         yield {
             'name': record.name,
@@ -609,6 +776,8 @@ class ConstellixProvider(BaseProvider):
         }
 
     def _handle_pools(self, record):
+        healthcheck = self._healthcheck_config(record)
+
         # If we don't have dynamic, then there's no pools
         if not getattr(record, 'dynamic', False):
             return []
@@ -629,6 +798,26 @@ class ConstellixProvider(BaseProvider):
             generated_pool_name = \
                 f'{record.zone.name}:{record.name}:{record._type}:{pool_name}'
 
+            # Create Sonar checks if needed
+            if healthcheck is not None:
+                check_sites = self._sonar.\
+                    agents_for_regions(healthcheck["sonar_regions"])
+                for value in values:
+                    check_obj = self._create_update_check(
+                        pool_type = record._type,
+                        check_name = '{}-{}'.format(
+                            generated_pool_name,
+                            value['value']
+                        ),
+                        check_type = healthcheck["sonar_type"].lower(),
+                        value = value['value'],
+                        port = healthcheck["sonar_port"],
+                        interval = healthcheck["sonar_interval"],
+                        sites = check_sites
+                    )
+                    value['checkId'] = check_obj['id']
+                    value['policy'] = "followsonar"
+
             # OK, pool is valid, let's create it or update it
             self.log.debug("Creating pool %s", generated_pool_name)
             pool_obj = self._create_update_pool(
@@ -677,6 +866,37 @@ class ConstellixProvider(BaseProvider):
             res_pools.append(pool_obj)
         return res_pools
 
+    def _create_update_check(
+            self,
+            pool_type,
+            check_name,
+            check_type,
+            value,
+            port,
+            interval,
+            sites):
+
+        check = {
+            'name': check_name,
+            'host': value,
+            'port': port,
+            'checkSites': sites,
+            'interval': interval
+        }
+        if pool_type == "AAAA":
+            check['ipVersion'] = "IPV6"
+        else:
+            check['ipVersion'] = "IPV4"
+
+        if check_type == "http":
+            check['protocolType'] = "HTTPS"
+
+        existing_check = self._sonar.check(check_type, check_name)
+        if existing_check:
+            self._sonar.check_delete(existing_check['id'])
+
+        return self._sonar.check_create(check_type, check)
+
     def _create_update_pool(self, pool_name, pool_type, ttl, values):
         pool = {
             'name': pool_name,

tests/test_octodns_provider_constellix.py

@@ -6,7 +6,7 @@
 from __future__ import absolute_import, division, print_function, \
     unicode_literals
 
-from mock import Mock, call
+from mock import Mock, PropertyMock, call
 from os.path import dirname, join
 from requests import HTTPError
 from requests_mock import ANY, mock as requests_mock
@@ -72,6 +72,141 @@ class TestConstellixProvider(TestCase):
             expected._remove_record(record)
             break
 
+    expected_healthcheck = Zone('unit.tests.', [])
+    source = YamlProvider('test', join(dirname(__file__), 'config'))
+    source.populate(expected_healthcheck)
+
+    # Our test suite differs a bit, add our NS and remove the simple one
+    expected_healthcheck.add_record(Record.new(expected_healthcheck, 'under', {
+        'ttl': 3600,
+        'type': 'NS',
+        'values': [
+            'ns1.unit.tests.',
+            'ns2.unit.tests.',
+        ]
+    }))
+
+    # Add some ALIAS records
+    expected_healthcheck.add_record(Record.new(expected_healthcheck, '', {
+        'ttl': 1800,
+        'type': 'ALIAS',
+        'value': 'aname.unit.tests.'
+    }))
+
+    # Add a dynamic record
+    expected_healthcheck.add_record(
+        Record.new(expected_healthcheck, 'www.dynamic', {
+            'ttl': 300,
+            'type': 'A',
+            'values': [
+                '1.2.3.4',
+                '1.2.3.5'
+            ],
+            'dynamic': {
+                'pools': {
+                    'two': {
+                        'values': [{
+                            'value': '1.2.3.4',
+                            'weight': 1
+                        }, {
+                            'value': '1.2.3.5',
+                            'weight': 1
+                        }],
+                    },
+                },
+                'rules': [{
+                    'pool': 'two',
+                }],
+            },
+            'octodns': {
+                'constellix': {
+                    'healthcheck': {
+                        'sonar_port': 80,
+                        'sonar_regions': [
+                            'ASIAPAC',
+                            'EUROPE'
+                        ],
+                        'sonar_type': 'TCP'
+                    }
+                }
+            }
+        })
+    )
+
+    for record in list(expected_healthcheck.records):
+        if record.name == 'sub' and record._type == 'NS':
+            expected_healthcheck._remove_record(record)
+            break
+
+    expected_healthcheck_world = Zone('unit.tests.', [])
+    source = YamlProvider('test', join(dirname(__file__), 'config'))
+    source.populate(expected_healthcheck_world)
+
+    # Our test suite differs a bit, add our NS and remove the simple one
+    expected_healthcheck_world.add_record(
+        Record.new(expected_healthcheck_world, 'under', {
+            'ttl': 3600,
+            'type': 'NS',
+            'values': [
+                'ns1.unit.tests.',
+                'ns2.unit.tests.',
+            ]
+        })
+    )
+
+    # Add some ALIAS records
+    expected_healthcheck_world.add_record(
+        Record.new(expected_healthcheck_world, '', {
+            'ttl': 1800,
+            'type': 'ALIAS',
+            'value': 'aname.unit.tests.'
+        })
+    )
+
+    # Add a dynamic record
+    expected_healthcheck_world.add_record(
+        Record.new(expected_healthcheck_world, 'www.dynamic', {
+            'ttl': 300,
+            'type': 'AAAA',
+            'values': [
+                '2601:644:500:e210:62f8:1dff:feb8:947a',
+                '2601:642:500:e210:62f8:1dff:feb8:947a'
+            ],
+            'dynamic': {
+                'pools': {
+                    'two': {
+                        'values': [{
+                            'value': '2601:644:500:e210:62f8:1dff:feb8:947a',
+                            'weight': 1
+                        }, {
+                            'value': '2601:642:500:e210:62f8:1dff:feb8:947a',
+                            'weight': 1
+                        }],
+                    },
+                },
+                'rules': [{
+                    'pool': 'two',
+                }],
+            },
+            'octodns': {
+                'constellix': {
+                    'healthcheck': {
+                        'sonar_port': 80,
+                        'sonar_regions': [
+                            'WORLD'
+                        ],
+                        'sonar_type': 'HTTP'
+                    }
+                }
+            }
+        })
+    )
+
+    for record in list(expected_healthcheck_world.records):
+        if record.name == 'sub' and record._type == 'NS':
+            expected_healthcheck_world._remove_record(record)
+            break
+
     expected_dynamic = Zone('unit.tests.', [])
     source = YamlProvider('test', join(dirname(__file__), 'config'))
     source.populate(expected_dynamic)
@@ -157,6 +292,14 @@ class TestConstellixProvider(TestCase):
                 provider.populate(zone)
             self.assertEquals('Unauthorized', str(ctx.exception))
 
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=401,
+                     text='{"errors": ["Unable to authenticate token"]}')
+
+            with self.assertRaises(Exception) as ctx:
+                provider._sonar.agents
+            self.assertEquals('Unauthorized', str(ctx.exception))
+
         # Bad request
         with requests_mock() as mock:
             mock.get(ANY, status_code=400,
@@ -169,6 +312,15 @@ class TestConstellixProvider(TestCase):
             self.assertEquals('\n  - "unittests" is not a valid domain name',
                               str(ctx.exception))
 
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=400,
+                     text='error text')
+
+            with self.assertRaises(Exception) as ctx:
+                provider._sonar.agents
+            self.assertEquals('\n  - error text',
+                              str(ctx.exception))
+
         # General error
         with requests_mock() as mock:
             mock.get(ANY, status_code=502, text='Things caught fire')
@@ -187,6 +339,19 @@ class TestConstellixProvider(TestCase):
             provider.populate(zone)
             self.assertEquals(set(), zone.records)
 
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=404, text='')
+            with self.assertRaises(Exception) as ctx:
+                provider._sonar.agents
+            self.assertEquals('Not Found', str(ctx.exception))
+
+        # Sonar Normal response
+        with requests_mock() as mock:
+            mock.get(ANY, status_code=200, text='[]')
+            agents = provider._sonar.agents
+            self.assertEquals({}, agents)
+            agents = provider._sonar.agents
+
         # No diffs == no changes
         with requests_mock() as mock:
             base = 'https://api.dns.constellix.com/v1'
@@ -426,7 +591,524 @@ class TestConstellixProvider(TestCase):
             call('DELETE', '/domains/123123/records/ANAME/11189899'),
         ], any_order=True)
 
-    def test_apply_dunamic(self):
+    def test_apply_healthcheck(self):
+        provider = ConstellixProvider('test', 'api', 'secret')
+
+        resp = Mock()
+        resp.json = Mock()
+        provider._client._request = Mock(return_value=resp)
+
+        # non-existent domain, create everything
+        resp.json.side_effect = [
+            [],  # no domains returned during populate
+            [{
+                'id': 123123,
+                'name': 'unit.tests'
+            }],  # domain created in apply
+            [],  # No pools returned during populate
+            [{
+                "id": 1808520,
+                "name": "unit.tests.:www.dynamic:A:two",
+            }]   # pool created in apply
+        ]
+
+        sonar_resp = Mock()
+        sonar_resp.json = Mock()
+        type(sonar_resp).headers = PropertyMock(return_value={
+            "Location": "http://api.sonar.constellix.com/rest/api/tcp/52906"
+        })
+        sonar_resp.headers = Mock()
+        provider._sonar._request = Mock(return_value=sonar_resp)
+
+        sonar_resp.json.side_effect = [
+            [{
+                "id": 1,
+                "name": "USWAS01",
+                "label": "Site 1",
+                "location": "Washington, DC, U.S.A",
+                "country": "U.S.A",
+                "region": "ASIAPAC"
+            }, {
+                "id": 23,
+                "name": "CATOR01",
+                "label": "Site 1",
+                "location": "Toronto,Canada",
+                "country": "Canada",
+                "region": "EUROPE"
+            }, {
+                "id": 25,
+                "name": "CATOR01",
+                "label": "Site 1",
+                "location": "Toronto,Canada",
+                "country": "Canada",
+                "region": "OCEANIA"
+            }],  # available agents
+            [{
+                "id": 52,
+                "name": "unit.tests.:www.dynamic:A:two-1.2.3.4"
+            }],  # initial checks
+            {
+                "type": 'TCP'
+            },  # check type
+            {
+                "id": 52906,
+                "name": "unit.tests.:www.dynamic:A:two-1.2.3.4"
+            },
+            {
+                "id": 52907,
+                "name": "unit.tests.:www.dynamic:A:two-1.2.3.5"
+            }
+        ]
+
+        plan = provider.plan(self.expected_healthcheck)
+
+        # No root NS, no ignored, no excluded, no unsupported
+        n = len(self.expected_healthcheck.records) - 8
+        self.assertEquals(n, len(plan.changes))
+        self.assertEquals(n, provider.apply(plan))
+
+        provider._client._request.assert_has_calls([
+            # get all domains to build the cache
+            call('GET', '/domains'),
+            # created the domain
+            call('POST', '/domains', data={'names': ['unit.tests']})
+        ])
+
+        # Check we tried to get our pool
+        provider._client._request.assert_has_calls([
+            # get all pools to build the cache
+            call('GET', '/pools/A'),
+            # created the pool
+            call('POST', '/pools/A', data={
+                'name': 'unit.tests.:www.dynamic:A:two',
+                'type': 'A',
+                'numReturn': 1,
+                'minAvailableFailover': 1,
+                'ttl': 300,
+                'values': [{
+                    "value": "1.2.3.4",
+                    "weight": 1,
+                    "checkId": 52906,
+                    "policy": 'followsonar'
+                }, {
+                    "value": "1.2.3.5",
+                    "weight": 1,
+                    "checkId": 52907,
+                    "policy": 'followsonar'
+                }]
+            })
+        ])
+
+        # These two checks are broken up so that ordering doesn't break things.
+        # Python3 doesn't make the calls in a consistent order so different
+        # things follow the GET / on different runs
+        provider._client._request.assert_has_calls([
+            call('POST', '/domains/123123/records/SRV', data={
+                'roundRobin': [{
+                    'priority': 10,
+                    'weight': 20,
+                    'value': 'foo-1.unit.tests.',
+                    'port': 30
+                }, {
+                    'priority': 12,
+                    'weight': 20,
+                    'value': 'foo-2.unit.tests.',
+                    'port': 30
+                }],
+                'name': '_srv._tcp',
+                'ttl': 600,
+            }),
+        ])
+
+        self.assertEquals(22, provider._client._request.call_count)
+
+        provider._client._request.reset_mock()
+
+        provider._client.records = Mock(return_value=[
+            {
+                'id': 11189897,
+                'type': 'A',
+                'name': 'www',
+                'ttl': 300,
+                'recordOption': 'roundRobin',
+                'value': [
+                    '1.2.3.4',
+                    '2.2.3.4',
+                ]
+            }, {
+                'id': 11189898,
+                'type': 'A',
+                'name': 'ttl',
+                'ttl': 600,
+                'recordOption': 'roundRobin',
+                'value': [
+                    '3.2.3.4'
+                ]
+            },  {
+                'id': 11189899,
+                'type': 'ALIAS',
+                'name': 'alias',
+                'ttl': 600,
+                'recordOption': 'roundRobin',
+                'value': [{
+                    'value': 'aname.unit.tests.'
+                }]
+            }, {
+                "id": 1808520,
+                "type": "A",
+                "name": "www.dynamic",
+                "geolocation": None,
+                "recordOption": "pools",
+                "ttl": 300,
+                "value": [],
+                "pools": [
+                    1808521
+                ]
+            }
+        ])
+
+        provider._client.pools = Mock(return_value=[{
+            "id": 1808521,
+            "name": "unit.tests.:www.dynamic:A:two",
+            "type": "A",
+            "values": [
+                {
+                    "value": "1.2.3.4",
+                    "weight": 1
+                },
+                {
+                    "value": "1.2.3.5",
+                    "weight": 1
+                }
+            ]
+        }])
+
+        # Domain exists, we don't care about return
+        resp.json.side_effect = [
+            [],  # no domains returned during populate
+            [{
+                'id': 123123,
+                'name': 'unit.tests'
+            }],  # domain created in apply
+            [],  # No pools returned during populate
+            [{
+                "id": 1808521,
+                "name": "unit.tests.:www.dynamic:A:one"
+            }]  # pool created in apply
+        ]
+
+        wanted = Zone('unit.tests.', [])
+        wanted.add_record(Record.new(wanted, 'ttl', {
+            'ttl': 300,
+            'type': 'A',
+            'value': '3.2.3.4'
+        }))
+
+        wanted.add_record(Record.new(wanted, 'www.dynamic', {
+            'ttl': 300,
+            'type': 'A',
+            'values': [
+                '1.2.3.4'
+            ],
+            'dynamic': {
+                'pools': {
+                    'two': {
+                        'values': [{
+                            'value': '1.2.3.4',
+                            'weight': 1
+                        }],
+                    },
+                },
+                'rules': [{
+                    'pool': 'two',
+                }],
+            },
+        }))
+
+        plan = provider.plan(wanted)
+        self.assertEquals(4, len(plan.changes))
+        self.assertEquals(4, provider.apply(plan))
+
+        # recreate for update, and deletes for the 2 parts of the other
+        provider._client._request.assert_has_calls([
+            call('POST', '/domains/123123/records/A', data={
+                'roundRobin': [{
+                    'value': '3.2.3.4'
+                }],
+                'name': 'ttl',
+                'ttl': 300
+            }),
+            call('PUT', '/pools/A/1808521', data={
+                'name': 'unit.tests.:www.dynamic:A:two',
+                'type': 'A',
+                'numReturn': 1,
+                'minAvailableFailover': 1,
+                'ttl': 300,
+                'values': [{
+                    "value": "1.2.3.4",
+                    "weight": 1
+                }],
+                'id': 1808521,
+                'geofilter': 1
+            }),
+            call('DELETE', '/domains/123123/records/A/11189897'),
+            call('DELETE', '/domains/123123/records/A/11189898'),
+            call('DELETE', '/domains/123123/records/ANAME/11189899'),
+        ], any_order=True)
+
+    def test_apply_healthcheck_world(self):
+        provider = ConstellixProvider('test', 'api', 'secret')
+
+        resp = Mock()
+        resp.json = Mock()
+        provider._client._request = Mock(return_value=resp)
+
+        # non-existent domain, create everything
+        resp.json.side_effect = [
+            [],  # no domains returned during populate
+            [{
+                'id': 123123,
+                'name': 'unit.tests'
+            }],  # domain created in apply
+            [],  # No pools returned during populate
+            [{
+                "id": 1808520,
+                "name": "unit.tests.:www.dynamic:A:two",
+            }]   # pool created in apply
+        ]
+
+        sonar_resp = Mock()
+        sonar_resp.json = Mock()
+        type(sonar_resp).headers = PropertyMock(return_value={
+            "Location": "http://api.sonar.constellix.com/rest/api/tcp/52906"
+        })
+        sonar_resp.headers = Mock()
+        provider._sonar._request = Mock(return_value=sonar_resp)
+
+        sonar_resp.json.side_effect = [
+            [{
+                "id": 1,
+                "name": "USWAS01",
+                "label": "Site 1",
+                "location": "Washington, DC, U.S.A",
+                "country": "U.S.A",
+                "region": "ASIAPAC"
+            }, {
+                "id": 23,
+                "name": "CATOR01",
+                "label": "Site 1",
+                "location": "Toronto,Canada",
+                "country": "Canada",
+                "region": "EUROPE"
+            }],  # available agents
+            [],  # no checks
+            {
+                "id": 52906,
+                "name": "check1"
+            },
+            {
+                "id": 52907,
+                "name": "check2"
+            }
+        ]
+
+        plan = provider.plan(self.expected_healthcheck_world)
+
+        # No root NS, no ignored, no excluded, no unsupported
+        n = len(self.expected_healthcheck.records) - 8
+        self.assertEquals(n, len(plan.changes))
+        self.assertEquals(n, provider.apply(plan))
+
+        provider._client._request.assert_has_calls([
+            # get all domains to build the cache
+            call('GET', '/domains'),
+            # created the domain
+            call('POST', '/domains', data={'names': ['unit.tests']})
+        ])
+
+        # Check we tried to get our pool
+        provider._client._request.assert_has_calls([
+            # get all pools to build the cache
+            call('GET', '/pools/AAAA'),
+            # created the pool
+            call('POST', '/pools/AAAA', data={
+                'name': 'unit.tests.:www.dynamic:AAAA:two',
+                'type': 'AAAA',
+                'numReturn': 1,
+                'minAvailableFailover': 1,
+                'ttl': 300,
+                'values': [{
+                    "value": "2601:642:500:e210:62f8:1dff:feb8:947a",
+                    "weight": 1,
+                    "checkId": 52906,
+                    "policy": 'followsonar'
+                }, {
+                    "value": "2601:644:500:e210:62f8:1dff:feb8:947a",
+                    "weight": 1,
+                    "checkId": 52907,
+                    "policy": 'followsonar'
+                }]
+            })
+        ])
+
+        # These two checks are broken up so that ordering doesn't break things.
+        # Python3 doesn't make the calls in a consistent order so different
+        # things follow the GET / on different runs
+        provider._client._request.assert_has_calls([
+            call('POST', '/domains/123123/records/SRV', data={
+                'roundRobin': [{
+                    'priority': 10,
+                    'weight': 20,
+                    'value': 'foo-1.unit.tests.',
+                    'port': 30
+                }, {
+                    'priority': 12,
+                    'weight': 20,
+                    'value': 'foo-2.unit.tests.',
+                    'port': 30
+                }],
+                'name': '_srv._tcp',
+                'ttl': 600,
+            }),
+        ])
+
+        self.assertEquals(22, provider._client._request.call_count)
+
+        provider._client._request.reset_mock()
+
+        provider._client.records = Mock(return_value=[
+            {
+                'id': 11189897,
+                'type': 'A',
+                'name': 'www',
+                'ttl': 300,
+                'recordOption': 'roundRobin',
+                'value': [
+                    '1.2.3.4',
+                    '2.2.3.4',
+                ]
+            }, {
+                'id': 11189898,
+                'type': 'A',
+                'name': 'ttl',
+                'ttl': 600,
+                'recordOption': 'roundRobin',
+                'value': [
+                    '3.2.3.4'
+                ]
+            },  {
+                'id': 11189899,
+                'type': 'ALIAS',
+                'name': 'alias',
+                'ttl': 600,
+                'recordOption': 'roundRobin',
+                'value': [{
+                    'value': 'aname.unit.tests.'
+                }]
+            }, {
+                "id": 1808520,
+                "type": "A",
+                "name": "www.dynamic",
+                "geolocation": None,
+                "recordOption": "pools",
+                "ttl": 300,
+                "value": [],
+                "pools": [
+                    1808521
+                ]
+            }
+        ])
+
+        provider._client.pools = Mock(return_value=[{
+            "id": 1808521,
+            "name": "unit.tests.:www.dynamic:A:two",
+            "type": "A",
+            "values": [
+                {
+                    "value": "1.2.3.4",
+                    "weight": 1
+                },
+                {
+                    "value": "1.2.3.5",
+                    "weight": 1
+                }
+            ]
+        }])
+
+        # Domain exists, we don't care about return
+        resp.json.side_effect = [
+            [],  # no domains returned during populate
+            [{
+                'id': 123123,
+                'name': 'unit.tests'
+            }],  # domain created in apply
+            [],  # No pools returned during populate
+            [{
+                "id": 1808521,
+                "name": "unit.tests.:www.dynamic:A:one"
+            }]  # pool created in apply
+        ]
+
+        wanted = Zone('unit.tests.', [])
+        wanted.add_record(Record.new(wanted, 'ttl', {
+            'ttl': 300,
+            'type': 'A',
+            'value': '3.2.3.4'
+        }))
+
+        wanted.add_record(Record.new(wanted, 'www.dynamic', {
+            'ttl': 300,
+            'type': 'A',
+            'values': [
+                '1.2.3.4'
+            ],
+            'dynamic': {
+                'pools': {
+                    'two': {
+                        'values': [{
+                            'value': '1.2.3.4',
+                            'weight': 1
+                        }],
+                    },
+                },
+                'rules': [{
+                    'pool': 'two',
+                }],
+            },
+        }))
+
+        plan = provider.plan(wanted)
+        self.assertEquals(4, len(plan.changes))
+        self.assertEquals(4, provider.apply(plan))
+
+        # recreate for update, and deletes for the 2 parts of the other
+        provider._client._request.assert_has_calls([
+            call('POST', '/domains/123123/records/A', data={
+                'roundRobin': [{
+                    'value': '3.2.3.4'
+                }],
+                'name': 'ttl',
+                'ttl': 300
+            }),
+            call('PUT', '/pools/A/1808521', data={
+                'name': 'unit.tests.:www.dynamic:A:two',
+                'type': 'A',
+                'numReturn': 1,
+                'minAvailableFailover': 1,
+                'ttl': 300,
+                'values': [{
+                    "value": "1.2.3.4",
+                    "weight": 1
+                }],
+                'id': 1808521,
+                'geofilter': 1
+            }),
+            call('DELETE', '/domains/123123/records/A/11189897'),
+            call('DELETE', '/domains/123123/records/A/11189898'),
+            call('DELETE', '/domains/123123/records/ANAME/11189899'),
+        ], any_order=True)
+
+    def test_apply_dynamic(self):
         provider = ConstellixProvider('test', 'api', 'secret')
 
         resp = Mock()