From d688c6123ad7c53f9b961f5eb73d44db34c8a120 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Wed, 3 Mar 2021 18:54:19 +0300 Subject: [PATCH] Added G-Core DNS API v2 provider with support of A, AAAA records --- README.md | 1 + octodns/provider/gcore.py | 229 ++++++++++++++++++++++++ tests/fixtures/gcore-no-changes.json | 53 ++++++ tests/fixtures/gcore-records.json | 25 +++ tests/fixtures/gcore-zone.json | 27 +++ tests/test_octodns_provider_gcore.py | 251 +++++++++++++++++++++++++++ 6 files changed, 586 insertions(+) create mode 100644 octodns/provider/gcore.py create mode 100644 tests/fixtures/gcore-no-changes.json create mode 100644 tests/fixtures/gcore-records.json create mode 100644 tests/fixtures/gcore-zone.json create mode 100644 tests/test_octodns_provider_gcore.py diff --git a/README.md b/README.md index a65e28f..0be2427 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | +| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | Missing `NA` geo target | diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py new file mode 100644 index 0000000..42e2cb7 --- /dev/null +++ b/octodns/provider/gcore.py @@ -0,0 +1,229 @@ +# +# +# + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from collections import defaultdict +from requests import Session +import logging + +from ..record import Record +from .base import BaseProvider + + +class GCoreClientException(Exception): + def __init__(self, r): + super(GCoreClientException, self).__init__(r.text) + + +class GCoreClientBadRequest(GCoreClientException): + def __init__(self, r): + super(GCoreClientBadRequest, self).__init__(r) + + +class GCoreClientNotFound(GCoreClientException): + def __init__(self, r): + super(GCoreClientNotFound, self).__init__(r) + + +class GCoreClient(object): + + ROOT_ZONES = "/zones" + + def __init__(self, base_url, token): + session = Session() + session.headers.update({"Authorization": "Bearer {}".format(token)}) + self._session = session + self._base_url = base_url + + def _request(self, method, path, params={}, data=None): + url = "{}{}".format(self._base_url, path) + r = self._session.request( + method, url, params=params, json=data, timeout=30.0 + ) + if r.status_code == 400: + raise GCoreClientBadRequest(r) + elif r.status_code == 404: + raise GCoreClientNotFound(r) + elif r.status_code == 500: + raise GCoreClientException(r) + r.raise_for_status() + return r + + def zone(self, zone_name): + return self._request( + "GET", "{}/{}".format(self.ROOT_ZONES, zone_name) + ).json() + + def zone_create(self, zone_name): + return self._request( + "POST", self.ROOT_ZONES, data={"name": zone_name} + ).json() + + def zone_records(self, zone_name): + rrsets = self._request( + "GET", "{}/{}/rrsets".format(self.ROOT_ZONES, zone_name) + ).json() + records = rrsets["rrsets"] + return records + + def record_create(self, zone_name, rrset_name, type_, data): + self._request( + "POST", self._rrset_url(zone_name, rrset_name, type_), data=data + ) + + def record_update(self, zone_name, rrset_name, type_, data): + self._request( + "PUT", self._rrset_url(zone_name, rrset_name, type_), data=data + ) + + def record_delete(self, zone_name, rrset_name, type_): + self._request("DELETE", self._rrset_url(zone_name, rrset_name, type_)) + + def _rrset_url(self, zone_name, rrset_name, type_): + return "{}/{}/{}/{}".format( + self.ROOT_ZONES, zone_name, rrset_name, type_ + ) + + +class GCoreProvider(BaseProvider): + """ + GCore provider using API v2. + + gcore: + class: octodns.provider.gcore.GCoreProvider + # Your API key (required) + token: XXXXXXXXXXXX + # url: https://dnsapi.gcorelabs.com/v2 + """ + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(("A", "AAAA")) + + def __init__(self, id, token, *args, **kwargs): + base_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") + self.log = logging.getLogger("GCoreProvider[{}]".format(id)) + self.log.debug("__init__: id=%s, token=***", id) + super(GCoreProvider, self).__init__(id, *args, **kwargs) + self._client = GCoreClient(base_url, token) + + def _data_for_single(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": record["resource_records"][0]["content"], + } + + _data_for_A = _data_for_single + _data_for_AAAA = _data_for_single + + def zone_records(self, zone): + try: + return self._client.zone_records(zone.name[:-1]), True + except GCoreClientNotFound: + return [], False + + def populate(self, zone, target=False, lenient=False): + self.log.debug( + "populate: name=%s, target=%s, lenient=%s", + zone.name, + target, + lenient, + ) + + values = defaultdict(defaultdict) + records, exists = self.zone_records(zone) + for record in records: + _type = record["type"] + if _type not in self.SUPPORTS: + continue + rr_name = record["name"].replace(zone.name, "") + if len(rr_name) > 0 and rr_name.endswith("."): + rr_name = rr_name[:-1] + values[rr_name][_type] = record + + before = len(zone.records) + for name, types in values.items(): + for _type, record in types.items(): + data_for = getattr(self, "_data_for_{}".format(_type)) + record = Record.new( + zone, + name, + data_for(_type, record), + source=self, + lenient=lenient, + ) + zone.add_record(record, lenient=lenient) + + self.log.info( + "populate: found %s records, exists=%s", + len(zone.records) - before, + exists, + ) + return exists + + def _params_for_single(self, record): + return { + "ttl": record.ttl, + "resource_records": [{"content": record.values}], + } + + _params_for_A = _params_for_single + _params_for_AAAA = _params_for_single + + def _apply_create(self, change): + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_create( + new.zone.name[:-1], rrset_name, new._type, data + ) + + def _apply_update(self, change): + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_update( + new.zone.name[:-1], rrset_name, new._type, data + ) + + def _apply_delete(self, change): + existing = change.existing + rrset_name = self._build_rrset_name(existing) + self._client.record_delete( + existing.zone.name[:-1], rrset_name, existing._type + ) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + zone = desired.name[:-1] + self.log.debug( + "_apply: zone=%s, len(changes)=%d", desired.name, len(changes) + ) + + try: + self._client.zone(zone) + except GCoreClientNotFound: + self.log.info("_apply: no existing zone, trying to create it") + self._client.zone_create(zone) + self.log.info("_apply: zone has been successfully created") + + changes.reverse() + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, "_apply_{}".format(class_name.lower()))(change) + + @staticmethod + def _build_rrset_name(record): + if len(record.name) > 0: + return "{}.{}".format(record.name, record.zone.name) + return record.zone.name diff --git a/tests/fixtures/gcore-no-changes.json b/tests/fixtures/gcore-no-changes.json new file mode 100644 index 0000000..ad62c94 --- /dev/null +++ b/tests/fixtures/gcore-no-changes.json @@ -0,0 +1,53 @@ +{ + "rrsets": [{ + "name": "unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "1.2.3.4", + "1.2.3.5" + ] + }] + }, { + "name": "aaaa.unit.tests.", + "type": "AAAA", + "ttl": 600, + "resource_records": [{ + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + }] + }, { + "name": "www.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "2.2.3.6" + ] + }] + }, { + "name": "www.sub.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "2.2.3.6" + ] + }] + }, { + "name": "unit.tests.", + "type": "ns", + "ttl": 300, + "resource_records": [{ + "content": [ + "ns2.gcdn.services" + ] + }, { + "content": [ + "ns1.gcorelabs.net" + ] + }] + }] +} diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json new file mode 100644 index 0000000..4d4685e --- /dev/null +++ b/tests/fixtures/gcore-records.json @@ -0,0 +1,25 @@ +{ + "rrsets": [{ + "name": "unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "1.2.3.4" + ] + }] + }, { + "name": "unit.tests.", + "type": "ns", + "ttl": 300, + "resource_records": [{ + "content": [ + "ns2.gcdn.services" + ] + }, { + "content": [ + "ns1.gcorelabs.net" + ] + }] + }] +} diff --git a/tests/fixtures/gcore-zone.json b/tests/fixtures/gcore-zone.json new file mode 100644 index 0000000..7f70275 --- /dev/null +++ b/tests/fixtures/gcore-zone.json @@ -0,0 +1,27 @@ +{ + "id": 27757, + "name": "unit.test", + "nx_ttl": 300, + "retry": 5400, + "refresh": 0, + "expiry": 1209600, + "contact": "support@gcorelabs.com", + "serial": 1614752868, + "primary_server": "ns1.gcorelabs.net", + "records": [ + { + "id": 12419, + "name": "unit.test.", + "type": "ns", + "ttl": 300, + "short_answers": [ + "[ns2.gcdn.services]", + "[ns1.gcorelabs.net]" + ] + } + ], + "dns_servers": [ + "ns1.gcorelabs.net", + "ns2.gcdn.services" + ] +} \ No newline at end of file diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py new file mode 100644 index 0000000..74762da --- /dev/null +++ b/tests/test_octodns_provider_gcore.py @@ -0,0 +1,251 @@ +# +# +# + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from mock import Mock, call +from os.path import dirname, join +from requests_mock import ANY, mock as requests_mock +from six import text_type +from unittest import TestCase + +from octodns.record import Record, Update, Delete +from octodns.provider.gcore import ( + GCoreProvider, + GCoreClientBadRequest, + GCoreClientNotFound, + GCoreClientException, +) +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestGCoreProvider(TestCase): + expected = Zone("unit.tests.", []) + source = YamlProvider("test", join(dirname(__file__), "config")) + source.populate(expected) + + def test_populate(self): + + provider = GCoreProvider("test_id", "token") + + # 400 - Bad Request. + with requests_mock() as mock: + mock.get(ANY, status_code=400, text='{"error":"bad body"}') + + with self.assertRaises(GCoreClientBadRequest) as ctx: + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertIn('"error":"bad body"', text_type(ctx.exception)) + + # 404 - Not Found. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + + with self.assertRaises(GCoreClientNotFound) as ctx: + zone = Zone("unit.tests.", []) + provider._client.zone(zone) + self.assertIn( + '"error":"zone is not found"', text_type(ctx.exception) + ) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=500, text="Things caught fire") + + with self.assertRaises(GCoreClientException) as ctx: + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEquals("Things caught fire", text_type(ctx.exception)) + + # No diffs == no changes + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + with open("tests/fixtures/gcore-no-changes.json") as fh: + mock.get(base, text=fh.read()) + + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEquals(4, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 3 removed + 1 modified + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + with open("tests/fixtures/gcore-records.json") as fh: + mock.get(base, text=fh.read()) + + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEquals(1, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(4, len(changes)) + self.assertEquals( + 3, len([c for c in changes if isinstance(c, Delete)]) + ) + self.assertEquals( + 1, len([c for c in changes if isinstance(c, Update)]) + ) + + def test_apply(self): + provider = GCoreProvider("test_id", "token") + + # Zone does not exists but can be created. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + mock.post(ANY, status_code=200, text='{"id":1234}') + + plan = provider.plan(self.expected) + provider.apply(plan) + + # Zone does not exists and can't be created. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + mock.post( + ANY, + status_code=400, + text='{"error":"parent zone is already' + ' occupied by another client"}', + ) + + with self.assertRaises( + (GCoreClientNotFound, GCoreClientBadRequest) + ) as ctx: + plan = provider.plan(self.expected) + provider.apply(plan) + self.assertIn( + "parent zone is already occupied by another client", + text_type(ctx.exception), + ) + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + with open("tests/fixtures/gcore-zone.json") as fh: + zone = fh.read() + + # non-existent domain + resp.json.side_effect = [ + GCoreClientNotFound(resp), # no zone in populate + GCoreClientNotFound(resp), # no domain during apply + zone, + ] + plan = provider.plan(self.expected) + + # create all + self.assertEquals(4, len(plan.changes)) + self.assertEquals(4, provider.apply(plan)) + self.assertFalse(plan.exists) + + provider._client._request.assert_has_calls( + [ + call("GET", "/zones/unit.tests/rrsets"), + call("GET", "/zones/unit.tests"), + call("POST", "/zones", data={"name": "unit.tests"}), + call( + "POST", + "/zones/unit.tests/www.sub.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["2.2.3.6"]}], + }, + ), + call( + "POST", + "/zones/unit.tests/www.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["2.2.3.6"]}], + }, + ), + call( + "POST", + "/zones/unit.tests/aaaa.unit.tests./AAAA", + data={ + "ttl": 600, + "resource_records": [ + { + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + } + ], + }, + ), + call( + "POST", + "/zones/unit.tests/unit.tests./A", + data={ + "ttl": 300, + "resource_records": [ + {"content": ["1.2.3.4", "1.2.3.5"]} + ], + }, + ), + ] + ) + # expected number of total calls + self.assertEquals(7, provider._client._request.call_count) + + provider._client._request.reset_mock() + + # delete 1 and update 1 + provider._client.zone_records = Mock( + return_value=[ + { + "name": "www", + "ttl": 300, + "type": "A", + "resource_records": [{"content": ["1.2.3.4"]}], + }, + { + "name": "ttl", + "ttl": 600, + "type": "A", + "resource_records": [{"content": ["3.2.3.4"]}], + }, + ] + ) + + # Domain exists, we don't care about return + resp.json.side_effect = ["{}"] + + wanted = Zone("unit.tests.", []) + wanted.add_record( + Record.new( + wanted, "ttl", {"ttl": 300, "type": "A", "value": "3.2.3.4"} + ) + ) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + + provider._client._request.assert_has_calls( + [ + call("DELETE", "/zones/unit.tests/www.unit.tests./A"), + call( + "PUT", + "/zones/unit.tests/ttl.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["3.2.3.4"]}], + }, + ), + ] + )