Merge pull request #815 from maikelpoot/transip_provider_restclient_tino

TransIP: Update to REST API

README.md

@@ -212,6 +212,7 @@ The above command pulled the existing data out of Route53 and placed the results
 | [Rackspace](/octodns/provider/rackspace.py) | | A, AAAA, ALIAS, CNAME, MX, NS, PTR, SPF, TXT | No |  |
 | [Route53](/octodns/provider/route53.py) | boto3 | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | Both | CNAME health checks don't support a Host header |
 | [Selectel](/octodns/provider/selectel.py) | | A, AAAA, CNAME, MX, NS, SPF, SRV, TXT | No | |
+| [Transip](/octodns/provider/transip.py) | transip | A, AAAA, CNAME, MX, NS, SRV, SPF, TXT, SSHFP, CAA | No | |
 | [UltraDns](/octodns/provider/ultra.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | |
 | [AxfrSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, LOC, MX, NS, PTR, SPF, SRV, TXT | No | read-only |
 | [ZoneFileSource](/octodns/source/axfr.py) | | A, AAAA, CAA, CNAME, MX, NS, PTR, SPF, SRV, TXT | No | read-only |

octodns/provider/transip.py

@@ -0,0 +1,258 @@
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+from collections import defaultdict, namedtuple
+from logging import getLogger
+
+from transip import TransIP
+from transip.exceptions import TransIPHTTPError
+from transip.v6.objects import DnsEntry
+
+from . import ProviderException
+from ..record import Record
+from .base import BaseProvider
+
+DNSEntry = namedtuple('DNSEntry', ('name', 'expire', 'type', 'content'))
+
+
+class TransipException(ProviderException):
+    pass
+
+
+class TransipConfigException(TransipException):
+    pass
+
+
+class TransipNewZoneException(TransipException):
+    pass
+
+
+class TransipProvider(BaseProvider):
+    '''
+    Transip DNS provider
+
+    transip:
+        class: octodns.provider.transip.TransipProvider
+        # Your Transip account name (required)
+        account: yourname
+        # Path to a private key file (required if key is not used)
+        key_file: /path/to/file
+        # The api key as string (required if key_file is not used)
+        key: |
+            \'''
+            -----BEGIN PRIVATE KEY-----
+            ...
+            -----END PRIVATE KEY-----
+            \'''
+        # if both `key_file` and `key` are presented `key_file` is used
+
+    '''
+
+    SUPPORTS_GEO = False
+    SUPPORTS_DYNAMIC = False
+    SUPPORTS = set(('A', 'AAAA', 'CNAME', 'MX', 'NS', 'SRV', 'SPF', 'TXT',
+                    'SSHFP', 'CAA'))
+    # unsupported by OctoDNS: 'TLSA'
+    MIN_TTL = 120
+    TIMEOUT = 15
+    ROOT_RECORD = '@'
+
+    def __init__(self, id, account, key=None, key_file=None, *args, **kwargs):
+        self.log = getLogger('TransipProvider[{}]'.format(id))
+        self.log.debug('__init__: id=%s, account=%s, token=***', id,
+                       account)
+        super(TransipProvider, self).__init__(id, *args, **kwargs)
+
+        if key_file is not None:
+            self._client = TransIP(login=account, private_key_file=key_file)
+        elif key is not None:
+            self._client = TransIP(login=account, private_key=key)
+        else:
+            raise TransipConfigException(
+                'Missing `key` or `key_file` parameter in config'
+            )
+
+    def populate(self, zone, target=False, lenient=False):
+        '''
+        Populate the zone with records in-place.
+        '''
+        self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
+                       target, lenient)
+
+        before = len(zone.records)
+
+        try:
+            domain = self._client.domains.get(zone.name.strip('.'))
+            records = domain.dns.list()
+        except TransIPHTTPError as e:
+            if e.response_code == 404 and target is False:
+                # Zone not found in account, and not a target so just
+                # leave an empty zone.
+                return False
+            elif e.response_code == 404 and target is True:
+                self.log.warning('populate: Transip can\'t create new zones')
+                raise TransipNewZoneException(
+                    ('populate: ({}) Transip used ' +
+                     'as target for non-existing zone: {}').format(
+                        e.response_code, zone.name))
+            else:
+                self.log.error(
+                    'populate: (%s) %s ', e.response_code, e.message
+                )
+                raise TransipException(
+                    'Unhandled error: ({}) {}'.format(
+                        e.response_code, e.message
+                    )
+                )
+
+        self.log.debug(
+            'populate: found %s records for zone %s', len(records), zone.name
+        )
+        if records:
+            values = defaultdict(lambda: defaultdict(list))
+            for record in records:
+                name = zone.hostname_from_fqdn(record.name)
+                if name == self.ROOT_RECORD:
+                    name = ''
+
+                if record.type in self.SUPPORTS:
+                    values[name][record.type].append(record)
+
+            for name, types in values.items():
+                for _type, records in types.items():
+                    record = Record.new(
+                        zone,
+                        name,
+                        _data_for(_type, records, zone),
+                        source=self,
+                        lenient=lenient,
+                    )
+                    zone.add_record(record, lenient=lenient)
+        self.log.info('populate:   found %s records',
+                      len(zone.records) - before)
+
+        return True
+
+    def _apply(self, plan):
+        desired = plan.desired
+        changes = plan.changes
+        self.log.debug('apply: zone=%s, changes=%d', desired.name,
+                       len(changes))
+
+        try:
+            domain = self._client.domains.get(plan.desired.name[:-1])
+        except TransIPHTTPError as e:
+            self.log.exception('_apply: getting the domain failed')
+            raise TransipException(
+                'Unhandled error: ({}) {}'.format(e.response_code, e.message)
+            )
+
+        records = []
+        for record in plan.desired.records:
+            if record._type in self.SUPPORTS:
+                # Root records have '@' as name
+                name = record.name
+                if name == '':
+                    name = self.ROOT_RECORD
+
+                records.extend(_entries_for(name, record))
+
+        # Transform DNSEntry namedtuples into transip.v6.objects.DnsEntry
+        # objects, which is a bit ugly because it's quite a magical object.
+        api_records = [DnsEntry(domain.dns, r._asdict()) for r in records]
+        try:
+            domain.dns.replace(api_records)
+        except TransIPHTTPError as e:
+            self.log.warning(
+                '_apply: Set DNS returned one or more errors: {}'.format(e)
+            )
+            raise TransipException(
+                'Unhandled error: ({}) {}'.format(e.response_code, e.message)
+            )
+
+
+def _data_for(type_, records, current_zone):
+    if type_ == 'CNAME':
+        return {
+            'type': type_,
+            'ttl': records[0].expire,
+            'value': _parse_to_fqdn(records[0].content, current_zone),
+        }
+
+    def format_mx(record):
+        preference, exchange = record.content.split(' ', 1)
+        return {
+            'preference': preference,
+            'exchange': _parse_to_fqdn(exchange, current_zone),
+        }
+
+    def format_srv(record):
+        priority, weight, port, target = record.content.split(' ', 3)
+        return {
+            'port': port,
+            'priority': priority,
+            'target': _parse_to_fqdn(target, current_zone),
+            'weight': weight,
+        }
+
+    def format_sshfp(record):
+        algorithm, fp_type, fingerprint = record.content.split(' ', 2)
+        return {
+            'algorithm': algorithm,
+            'fingerprint': fingerprint.lower(),
+            'fingerprint_type': fp_type,
+        }
+
+    def format_caa(record):
+        flags, tag, value = record.content.split(' ', 2)
+        return {'flags': flags, 'tag': tag, 'value': value}
+
+    def format_txt(record):
+        return record.content.replace(';', '\\;')
+
+    value_formatter = {
+        'MX': format_mx,
+        'SRV': format_srv,
+        'SSHFP': format_sshfp,
+        'CAA': format_caa,
+        'TXT': format_txt,
+    }.get(type_, lambda r: r.content)
+
+    return {
+        'type': type_,
+        'ttl': _get_lowest_ttl(records),
+        'values': [value_formatter(r) for r in records],
+    }
+
+
+def _parse_to_fqdn(value, current_zone):
+    # TransIP allows '@' as value to alias the root record.
+    # this provider won't set an '@' value, but can be an existing record
+    if value == TransipProvider.ROOT_RECORD:
+        value = current_zone.name
+
+    if value[-1] != '.':
+        value = '{}.{}'.format(value, current_zone.name)
+
+    return value
+
+
+def _get_lowest_ttl(records):
+    return min([r.expire for r in records] + [100000])
+
+
+def _entries_for(name, record):
+    values = record.values if hasattr(record, 'values') else [record.value]
+    formatter = {
+        'MX': lambda v: f'{v.preference} {v.exchange}',
+        'SRV': lambda v: f'{v.priority} {v.weight} {v.port} {v.target}',
+        'SSHFP': lambda v: (
+            f'{v.algorithm} {v.fingerprint_type} {v.fingerprint}'
+        ),
+        'CAA': lambda v: f'{v.flags} {v.tag} {v.value}',
+        'TXT': lambda v: v.replace('\\;', ';'),
+    }.get(record._type, lambda r: r)
+    return [
+        DNSEntry(name, record.ttl, record._type, formatter(value))
+        for value in values
+    ]

requirements.txt

@@ -20,6 +20,7 @@ ovh==0.5.0
 pycountry-convert==0.7.2
 pycountry==20.7.3
 python-dateutil==2.8.1
-requests==2.24.0
+requests==2.25.1
 s3transfer==0.3.3
 setuptools==44.1.1
+python-transip==0.5.0

tests/test_octodns_provider_transip.py

@@ -0,0 +1,419 @@
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+from operator import itemgetter
+from os.path import dirname, join
+from unittest import TestCase
+from unittest.mock import Mock, patch
+
+from octodns.provider.transip import (DNSEntry, TransipConfigException,
+                                      TransipException,
+                                      TransipNewZoneException, TransipProvider,
+                                      _entries_for, _parse_to_fqdn)
+from octodns.provider.yaml import YamlProvider
+from octodns.zone import Zone
+from transip.exceptions import TransIPHTTPError
+
+
+def make_expected():
+    expected = Zone("unit.tests.", [])
+    source = YamlProvider("test", join(dirname(__file__), "config"))
+    source.populate(expected)
+    return expected
+
+
+def make_mock():
+    zone = make_expected()
+
+    # Turn Zone.records into TransIP DNSEntries
+    api_entries = []
+    for record in zone.records:
+        if record._type in TransipProvider.SUPPORTS:
+            # Root records have '@' as name
+            name = record.name
+            if name == "":
+                name = TransipProvider.ROOT_RECORD
+
+            api_entries.extend(_entries_for(name, record))
+
+    # Append bogus entry so test for record type not being in SUPPORTS is
+    # executed. For 100% test coverage.
+    api_entries.append(DNSEntry("@", "3600", "BOGUS", "ns.transip.nl"))
+
+    return zone, api_entries
+
+
+def make_mock_empty():
+    mock = Mock()
+    mock.return_value.domains.get.return_value.dns.list.return_value = []
+    return mock
+
+
+def make_failing_mock(response_code):
+    mock = Mock()
+    mock.return_value.domains.get.side_effect = [
+        TransIPHTTPError(str(response_code), response_code)
+    ]
+    return mock
+
+
+class TestTransipProvider(TestCase):
+
+    bogus_key = "-----BEGIN RSA PRIVATE KEY-----Z-----END RSA PRIVATE KEY-----"
+
+    @patch("octodns.provider.transip.TransIP", make_mock_empty())
+    def test_init(self):
+        with self.assertRaises(TransipConfigException) as ctx:
+            TransipProvider("test", "unittest")
+
+        self.assertEquals(
+            "Missing `key` or `key_file` parameter in config",
+            str(ctx.exception),
+        )
+
+        # Those should work
+        TransipProvider("test", "unittest", key=self.bogus_key)
+        TransipProvider("test", "unittest", key_file="/fake/path")
+
+    @patch("octodns.provider.transip.TransIP", make_failing_mock(401))
+    def test_populate_unauthenticated(self):
+        # Unhappy Plan - Not authenticated
+        provider = TransipProvider("test", "unittest", self.bogus_key)
+        zone = Zone("unit.tests.", [])
+        with self.assertRaises(TransipException):
+            provider.populate(zone, True)
+
+    @patch("octodns.provider.transip.TransIP", make_failing_mock(404))
+    def test_populate_new_zone_as_target(self):
+        # Unhappy Plan - Zone does not exists
+        # Will trigger an exception if provider is used as a target for a
+        # non-existing zone
+        provider = TransipProvider("test", "unittest", self.bogus_key)
+        zone = Zone("notfound.unit.tests.", [])
+        with self.assertRaises(TransipNewZoneException):
+            provider.populate(zone, True)
+
+    @patch("octodns.provider.transip.TransIP", make_mock_empty())
+    def test_populate_new_zone_not_target(self):
+        # Happy Plan - Zone does not exists
+        # Won't trigger an exception if provider is NOT used as a target for a
+        # non-existing zone.
+        provider = TransipProvider("test", "unittest", self.bogus_key)
+        zone = Zone("notfound.unit.tests.", [])
+        provider.populate(zone, False)
+
+    @patch("octodns.provider.transip.TransIP", make_failing_mock(404))
+    def test_populate_zone_does_not_exist(self):
+        # Happy Plan - Zone does not exists
+        # Won't trigger an exception if provider is NOT used as a target for a
+        # non-existing zone.
+        provider = TransipProvider("test", "unittest", self.bogus_key)
+        zone = Zone("notfound.unit.tests.", [])
+        provider.populate(zone, False)
+
+    @patch("octodns.provider.transip.TransIP")
+    def test_populate_zone_exists_not_target(self, mock_client):
+        # Happy Plan - Populate
+        source_zone, api_records = make_mock()
+        mock_client.return_value.domains.get.return_value.dns.list. \
+            return_value = api_records
+        provider = TransipProvider("test", "unittest", self.bogus_key)
+        zone = Zone("unit.tests.", [])
+
+        exists = provider.populate(zone, False)
+
+        self.assertTrue(exists, "populate should return True")
+
+        # Due to the implementation of Record._equality_tuple() we can't do a
+        # normal compare, as that ingores ttl's for example. We therefor use
+        # the __repr__ to compare. We do need to filter out `.geo` attributes
+        # that Transip doesn't support.
+        expected = set()
+        for r in source_zone.records:
+            if r._type in TransipProvider.SUPPORTS:
+                if hasattr(r, "geo"):
+                    r.geo = None
+                expected.add(r.__repr__())
+        self.assertEqual({r.__repr__() for r in zone.records}, expected)
+
+    @patch("octodns.provider.transip.TransIP", make_mock_empty())
+    def test_populate_zone_exists_as_target(self):
+        # Happy Plan - Even if the zone has no records the zone should exist
+        provider = TransipProvider("test", "unittest", self.bogus_key)
+        zone = Zone("unit.tests.", [])
+        exists = provider.populate(zone, True)
+        self.assertTrue(exists, "populate should return True")
+
+    @patch("octodns.provider.transip.TransIP", make_mock_empty())
+    def test_plan(self):
+        # Test happy plan, only create
+        provider = TransipProvider("test", "unittest", self.bogus_key)
+
+        plan = provider.plan(make_expected())
+
+        self.assertIsNotNone(plan)
+        self.assertEqual(15, plan.change_counts["Create"])
+        self.assertEqual(0, plan.change_counts["Update"])
+        self.assertEqual(0, plan.change_counts["Delete"])
+
+    @patch("octodns.provider.transip.TransIP")
+    def test_apply(self, client_mock):
+        # Test happy flow. Create all supported records
+        domain_mock = Mock()
+        client_mock.return_value.domains.get.return_value = domain_mock
+        domain_mock.dns.list.return_value = []
+        provider = TransipProvider("test", "unittest", self.bogus_key)
+
+        plan = provider.plan(make_expected())
+        self.assertIsNotNone(plan)
+        provider.apply(plan)
+
+        domain_mock.dns.replace.assert_called_once()
+
+        # These are the supported ones from tests/config/unit.test.yaml
+        expected_entries = [
+            {
+                "name": "ignored",
+                "expire": 3600,
+                "type": "A",
+                "content": "9.9.9.9",
+            },
+            {
+                "name": "@",
+                "expire": 3600,
+                "type": "CAA",
+                "content": "0 issue ca.unit.tests",
+            },
+            {
+                "name": "sub",
+                "expire": 3600,
+                "type": "NS",
+                "content": "6.2.3.4.",
+            },
+            {
+                "name": "sub",
+                "expire": 3600,
+                "type": "NS",
+                "content": "7.2.3.4.",
+            },
+            {
+                "name": "spf",
+                "expire": 600,
+                "type": "SPF",
+                "content": "v=spf1 ip4:192.168.0.1/16-all",
+            },
+            {
+                "name": "_srv._tcp",
+                "expire": 600,
+                "type": "SRV",
+                "content": "10 20 30 foo-1.unit.tests.",
+            },
+            {
+                "name": "_srv._tcp",
+                "expire": 600,
+                "type": "SRV",
+                "content": "12 20 30 foo-2.unit.tests.",
+            },
+            {
+                "name": "_pop3._tcp",
+                "expire": 600,
+                "type": "SRV",
+                "content": "0 0 0 .",
+            },
+            {
+                "name": "_imap._tcp",
+                "expire": 600,
+                "type": "SRV",
+                "content": "0 0 0 .",
+            },
+            {
+                "name": "txt",
+                "expire": 600,
+                "type": "TXT",
+                "content": "Bah bah black sheep",
+            },
+            {
+                "name": "txt",
+                "expire": 600,
+                "type": "TXT",
+                "content": "have you any wool.",
+            },
+            {
+                "name": "txt",
+                "expire": 600,
+                "type": "TXT",
+                "content": (
+                    "v=DKIM1;k=rsa;s=email;h=sha256;"
+                    "p=A/kinda+of/long/string+with+numb3rs"
+                ),
+            },
+            {"name": "@", "expire": 3600, "type": "NS", "content": "6.2.3.4."},
+            {"name": "@", "expire": 3600, "type": "NS", "content": "7.2.3.4."},
+            {
+                "name": "cname",
+                "expire": 300,
+                "type": "CNAME",
+                "content": "unit.tests.",
+            },
+            {
+                "name": "excluded",
+                "expire": 3600,
+                "type": "CNAME",
+                "content": "unit.tests.",
+            },
+            {
+                "name": "www.sub",
+                "expire": 300,
+                "type": "A",
+                "content": "2.2.3.6",
+            },
+            {
+                "name": "included",
+                "expire": 3600,
+                "type": "CNAME",
+                "content": "unit.tests.",
+            },
+            {
+                "name": "mx",
+                "expire": 300,
+                "type": "MX",
+                "content": "10 smtp-4.unit.tests.",
+            },
+            {
+                "name": "mx",
+                "expire": 300,
+                "type": "MX",
+                "content": "20 smtp-2.unit.tests.",
+            },
+            {
+                "name": "mx",
+                "expire": 300,
+                "type": "MX",
+                "content": "30 smtp-3.unit.tests.",
+            },
+            {
+                "name": "mx",
+                "expire": 300,
+                "type": "MX",
+                "content": "40 smtp-1.unit.tests.",
+            },
+            {
+                "name": "aaaa",
+                "expire": 600,
+                "type": "AAAA",
+                "content": "2601:644:500:e210:62f8:1dff:feb8:947a",
+            },
+            {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.4"},
+            {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.5"},
+            {"name": "www", "expire": 300, "type": "A", "content": "2.2.3.6"},
+            {
+                "name": "@",
+                "expire": 3600,
+                "type": "SSHFP",
+                "content": "1 1 7491973e5f8b39d5327cd4e08bc81b05f7710b49",
+            },
+            {
+                "name": "@",
+                "expire": 3600,
+                "type": "SSHFP",
+                "content": "1 1 bf6b6825d2977c511a475bbefb88aad54a92ac73",
+            },
+        ]
+        # Unpack from the transip library magic structure...
+        seen_entries = [
+            e.__dict__["_attrs"]
+            for e in domain_mock.dns.replace.mock_calls[0][1][0]
+        ]
+        self.assertEqual(
+            sorted(seen_entries, key=itemgetter("name", "type", "expire")),
+            sorted(expected_entries, key=itemgetter("name", "type", "expire")),
+        )
+
+    @patch("octodns.provider.transip.TransIP")
+    def test_apply_unsupported(self, client_mock):
+        # This triggers the if supported statement to give 100% code coverage
+        domain_mock = Mock()
+        client_mock.return_value.domains.get.return_value = domain_mock
+        domain_mock.dns.list.return_value = []
+        provider = TransipProvider("test", "unittest", self.bogus_key)
+
+        plan = provider.plan(make_expected())
+        self.assertIsNotNone(plan)
+
+        # Test apply with only support for A records
+        provider.SUPPORTS = set(("A"))
+
+        provider.apply(plan)
+        seen_entries = [
+            e.__dict__["_attrs"]
+            for e in domain_mock.dns.replace.mock_calls[0][1][0]
+        ]
+        expected_entries = [
+            {
+                "name": "ignored",
+                "expire": 3600,
+                "type": "A",
+                "content": "9.9.9.9",
+            },
+            {
+                "name": "www.sub",
+                "expire": 300,
+                "type": "A",
+                "content": "2.2.3.6",
+            },
+            {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.4"},
+            {"name": "@", "expire": 300, "type": "A", "content": "1.2.3.5"},
+            {"name": "www", "expire": 300, "type": "A", "content": "2.2.3.6"},
+        ]
+        self.assertEquals(
+            sorted(seen_entries, key=itemgetter("name", "type", "expire")),
+            sorted(expected_entries, key=itemgetter("name", "type", "expire")),
+        )
+
+    @patch("octodns.provider.transip.TransIP")
+    def test_apply_failure_on_not_found(self, client_mock):
+        # Test unhappy flow. Trigger 'not found error' in apply stage
+        # This should normally not happen as populate will capture it first
+        # but just in case.
+        domain_mock = Mock()
+        domain_mock.dns.list.return_value = []
+        client_mock.return_value.domains.get.side_effect = [
+            domain_mock,
+            TransIPHTTPError("Not Found", 404),
+        ]
+        provider = TransipProvider("test", "unittest", self.bogus_key)
+
+        plan = provider.plan(make_expected())
+
+        with self.assertRaises(TransipException):
+            provider.apply(plan)
+
+    @patch("octodns.provider.transip.TransIP")
+    def test_apply_failure_on_error(self, client_mock):
+        # Test unhappy flow. Trigger a unrecoverable error while saving
+        domain_mock = Mock()
+        domain_mock.dns.list.return_value = []
+        domain_mock.dns.replace.side_effect = [
+            TransIPHTTPError("Not Found", 500)
+        ]
+        client_mock.return_value.domains.get.return_value = domain_mock
+        provider = TransipProvider("test", "unittest", self.bogus_key)
+
+        plan = provider.plan(make_expected())
+
+        with self.assertRaises(TransipException):
+            provider.apply(plan)
+
+
+class TestParseFQDN(TestCase):
+    def test_parse_fqdn(self):
+        zone = Zone("unit.tests.", [])
+        self.assertEquals("www.unit.tests.", _parse_to_fqdn("www", zone))
+        self.assertEquals(
+            "www.unit.tests.", _parse_to_fqdn("www.unit.tests.", zone)
+        )
+        self.assertEquals(
+            "www.sub.sub.sub.unit.tests.",
+            _parse_to_fqdn("www.sub.sub.sub", zone),
+        )
+        self.assertEquals("unit.tests.", _parse_to_fqdn("@", zone))