From d688c6123ad7c53f9b961f5eb73d44db34c8a120 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Wed, 3 Mar 2021 18:54:19 +0300 Subject: [PATCH 01/14] Added G-Core DNS API v2 provider with support of A, AAAA records --- README.md | 1 + octodns/provider/gcore.py | 229 ++++++++++++++++++++++++ tests/fixtures/gcore-no-changes.json | 53 ++++++ tests/fixtures/gcore-records.json | 25 +++ tests/fixtures/gcore-zone.json | 27 +++ tests/test_octodns_provider_gcore.py | 251 +++++++++++++++++++++++++++ 6 files changed, 586 insertions(+) create mode 100644 octodns/provider/gcore.py create mode 100644 tests/fixtures/gcore-no-changes.json create mode 100644 tests/fixtures/gcore-records.json create mode 100644 tests/fixtures/gcore-zone.json create mode 100644 tests/test_octodns_provider_gcore.py diff --git a/README.md b/README.md index a65e28f..0be2427 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | +| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | Missing `NA` geo target | diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py new file mode 100644 index 0000000..42e2cb7 --- /dev/null +++ b/octodns/provider/gcore.py @@ -0,0 +1,229 @@ +# +# +# + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from collections import defaultdict +from requests import Session +import logging + +from ..record import Record +from .base import BaseProvider + + +class GCoreClientException(Exception): + def __init__(self, r): + super(GCoreClientException, self).__init__(r.text) + + +class GCoreClientBadRequest(GCoreClientException): + def __init__(self, r): + super(GCoreClientBadRequest, self).__init__(r) + + +class GCoreClientNotFound(GCoreClientException): + def __init__(self, r): + super(GCoreClientNotFound, self).__init__(r) + + +class GCoreClient(object): + + ROOT_ZONES = "/zones" + + def __init__(self, base_url, token): + session = Session() + session.headers.update({"Authorization": "Bearer {}".format(token)}) + self._session = session + self._base_url = base_url + + def _request(self, method, path, params={}, data=None): + url = "{}{}".format(self._base_url, path) + r = self._session.request( + method, url, params=params, json=data, timeout=30.0 + ) + if r.status_code == 400: + raise GCoreClientBadRequest(r) + elif r.status_code == 404: + raise GCoreClientNotFound(r) + elif r.status_code == 500: + raise GCoreClientException(r) + r.raise_for_status() + return r + + def zone(self, zone_name): + return self._request( + "GET", "{}/{}".format(self.ROOT_ZONES, zone_name) + ).json() + + def zone_create(self, zone_name): + return self._request( + "POST", self.ROOT_ZONES, data={"name": zone_name} + ).json() + + def zone_records(self, zone_name): + rrsets = self._request( + "GET", "{}/{}/rrsets".format(self.ROOT_ZONES, zone_name) + ).json() + records = rrsets["rrsets"] + return records + + def record_create(self, zone_name, rrset_name, type_, data): + self._request( + "POST", self._rrset_url(zone_name, rrset_name, type_), data=data + ) + + def record_update(self, zone_name, rrset_name, type_, data): + self._request( + "PUT", self._rrset_url(zone_name, rrset_name, type_), data=data + ) + + def record_delete(self, zone_name, rrset_name, type_): + self._request("DELETE", self._rrset_url(zone_name, rrset_name, type_)) + + def _rrset_url(self, zone_name, rrset_name, type_): + return "{}/{}/{}/{}".format( + self.ROOT_ZONES, zone_name, rrset_name, type_ + ) + + +class GCoreProvider(BaseProvider): + """ + GCore provider using API v2. + + gcore: + class: octodns.provider.gcore.GCoreProvider + # Your API key (required) + token: XXXXXXXXXXXX + # url: https://dnsapi.gcorelabs.com/v2 + """ + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = False + SUPPORTS = set(("A", "AAAA")) + + def __init__(self, id, token, *args, **kwargs): + base_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") + self.log = logging.getLogger("GCoreProvider[{}]".format(id)) + self.log.debug("__init__: id=%s, token=***", id) + super(GCoreProvider, self).__init__(id, *args, **kwargs) + self._client = GCoreClient(base_url, token) + + def _data_for_single(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": record["resource_records"][0]["content"], + } + + _data_for_A = _data_for_single + _data_for_AAAA = _data_for_single + + def zone_records(self, zone): + try: + return self._client.zone_records(zone.name[:-1]), True + except GCoreClientNotFound: + return [], False + + def populate(self, zone, target=False, lenient=False): + self.log.debug( + "populate: name=%s, target=%s, lenient=%s", + zone.name, + target, + lenient, + ) + + values = defaultdict(defaultdict) + records, exists = self.zone_records(zone) + for record in records: + _type = record["type"] + if _type not in self.SUPPORTS: + continue + rr_name = record["name"].replace(zone.name, "") + if len(rr_name) > 0 and rr_name.endswith("."): + rr_name = rr_name[:-1] + values[rr_name][_type] = record + + before = len(zone.records) + for name, types in values.items(): + for _type, record in types.items(): + data_for = getattr(self, "_data_for_{}".format(_type)) + record = Record.new( + zone, + name, + data_for(_type, record), + source=self, + lenient=lenient, + ) + zone.add_record(record, lenient=lenient) + + self.log.info( + "populate: found %s records, exists=%s", + len(zone.records) - before, + exists, + ) + return exists + + def _params_for_single(self, record): + return { + "ttl": record.ttl, + "resource_records": [{"content": record.values}], + } + + _params_for_A = _params_for_single + _params_for_AAAA = _params_for_single + + def _apply_create(self, change): + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_create( + new.zone.name[:-1], rrset_name, new._type, data + ) + + def _apply_update(self, change): + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_update( + new.zone.name[:-1], rrset_name, new._type, data + ) + + def _apply_delete(self, change): + existing = change.existing + rrset_name = self._build_rrset_name(existing) + self._client.record_delete( + existing.zone.name[:-1], rrset_name, existing._type + ) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + zone = desired.name[:-1] + self.log.debug( + "_apply: zone=%s, len(changes)=%d", desired.name, len(changes) + ) + + try: + self._client.zone(zone) + except GCoreClientNotFound: + self.log.info("_apply: no existing zone, trying to create it") + self._client.zone_create(zone) + self.log.info("_apply: zone has been successfully created") + + changes.reverse() + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, "_apply_{}".format(class_name.lower()))(change) + + @staticmethod + def _build_rrset_name(record): + if len(record.name) > 0: + return "{}.{}".format(record.name, record.zone.name) + return record.zone.name diff --git a/tests/fixtures/gcore-no-changes.json b/tests/fixtures/gcore-no-changes.json new file mode 100644 index 0000000..ad62c94 --- /dev/null +++ b/tests/fixtures/gcore-no-changes.json @@ -0,0 +1,53 @@ +{ + "rrsets": [{ + "name": "unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "1.2.3.4", + "1.2.3.5" + ] + }] + }, { + "name": "aaaa.unit.tests.", + "type": "AAAA", + "ttl": 600, + "resource_records": [{ + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + }] + }, { + "name": "www.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "2.2.3.6" + ] + }] + }, { + "name": "www.sub.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "2.2.3.6" + ] + }] + }, { + "name": "unit.tests.", + "type": "ns", + "ttl": 300, + "resource_records": [{ + "content": [ + "ns2.gcdn.services" + ] + }, { + "content": [ + "ns1.gcorelabs.net" + ] + }] + }] +} diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json new file mode 100644 index 0000000..4d4685e --- /dev/null +++ b/tests/fixtures/gcore-records.json @@ -0,0 +1,25 @@ +{ + "rrsets": [{ + "name": "unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [{ + "content": [ + "1.2.3.4" + ] + }] + }, { + "name": "unit.tests.", + "type": "ns", + "ttl": 300, + "resource_records": [{ + "content": [ + "ns2.gcdn.services" + ] + }, { + "content": [ + "ns1.gcorelabs.net" + ] + }] + }] +} diff --git a/tests/fixtures/gcore-zone.json b/tests/fixtures/gcore-zone.json new file mode 100644 index 0000000..7f70275 --- /dev/null +++ b/tests/fixtures/gcore-zone.json @@ -0,0 +1,27 @@ +{ + "id": 27757, + "name": "unit.test", + "nx_ttl": 300, + "retry": 5400, + "refresh": 0, + "expiry": 1209600, + "contact": "support@gcorelabs.com", + "serial": 1614752868, + "primary_server": "ns1.gcorelabs.net", + "records": [ + { + "id": 12419, + "name": "unit.test.", + "type": "ns", + "ttl": 300, + "short_answers": [ + "[ns2.gcdn.services]", + "[ns1.gcorelabs.net]" + ] + } + ], + "dns_servers": [ + "ns1.gcorelabs.net", + "ns2.gcdn.services" + ] +} \ No newline at end of file diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py new file mode 100644 index 0000000..74762da --- /dev/null +++ b/tests/test_octodns_provider_gcore.py @@ -0,0 +1,251 @@ +# +# +# + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from mock import Mock, call +from os.path import dirname, join +from requests_mock import ANY, mock as requests_mock +from six import text_type +from unittest import TestCase + +from octodns.record import Record, Update, Delete +from octodns.provider.gcore import ( + GCoreProvider, + GCoreClientBadRequest, + GCoreClientNotFound, + GCoreClientException, +) +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestGCoreProvider(TestCase): + expected = Zone("unit.tests.", []) + source = YamlProvider("test", join(dirname(__file__), "config")) + source.populate(expected) + + def test_populate(self): + + provider = GCoreProvider("test_id", "token") + + # 400 - Bad Request. + with requests_mock() as mock: + mock.get(ANY, status_code=400, text='{"error":"bad body"}') + + with self.assertRaises(GCoreClientBadRequest) as ctx: + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertIn('"error":"bad body"', text_type(ctx.exception)) + + # 404 - Not Found. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + + with self.assertRaises(GCoreClientNotFound) as ctx: + zone = Zone("unit.tests.", []) + provider._client.zone(zone) + self.assertIn( + '"error":"zone is not found"', text_type(ctx.exception) + ) + + # General error + with requests_mock() as mock: + mock.get(ANY, status_code=500, text="Things caught fire") + + with self.assertRaises(GCoreClientException) as ctx: + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEquals("Things caught fire", text_type(ctx.exception)) + + # No diffs == no changes + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + with open("tests/fixtures/gcore-no-changes.json") as fh: + mock.get(base, text=fh.read()) + + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEquals(4, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(0, len(changes)) + + # 3 removed + 1 modified + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + with open("tests/fixtures/gcore-records.json") as fh: + mock.get(base, text=fh.read()) + + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEquals(1, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEquals(4, len(changes)) + self.assertEquals( + 3, len([c for c in changes if isinstance(c, Delete)]) + ) + self.assertEquals( + 1, len([c for c in changes if isinstance(c, Update)]) + ) + + def test_apply(self): + provider = GCoreProvider("test_id", "token") + + # Zone does not exists but can be created. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + mock.post(ANY, status_code=200, text='{"id":1234}') + + plan = provider.plan(self.expected) + provider.apply(plan) + + # Zone does not exists and can't be created. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + mock.post( + ANY, + status_code=400, + text='{"error":"parent zone is already' + ' occupied by another client"}', + ) + + with self.assertRaises( + (GCoreClientNotFound, GCoreClientBadRequest) + ) as ctx: + plan = provider.plan(self.expected) + provider.apply(plan) + self.assertIn( + "parent zone is already occupied by another client", + text_type(ctx.exception), + ) + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + with open("tests/fixtures/gcore-zone.json") as fh: + zone = fh.read() + + # non-existent domain + resp.json.side_effect = [ + GCoreClientNotFound(resp), # no zone in populate + GCoreClientNotFound(resp), # no domain during apply + zone, + ] + plan = provider.plan(self.expected) + + # create all + self.assertEquals(4, len(plan.changes)) + self.assertEquals(4, provider.apply(plan)) + self.assertFalse(plan.exists) + + provider._client._request.assert_has_calls( + [ + call("GET", "/zones/unit.tests/rrsets"), + call("GET", "/zones/unit.tests"), + call("POST", "/zones", data={"name": "unit.tests"}), + call( + "POST", + "/zones/unit.tests/www.sub.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["2.2.3.6"]}], + }, + ), + call( + "POST", + "/zones/unit.tests/www.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["2.2.3.6"]}], + }, + ), + call( + "POST", + "/zones/unit.tests/aaaa.unit.tests./AAAA", + data={ + "ttl": 600, + "resource_records": [ + { + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + } + ], + }, + ), + call( + "POST", + "/zones/unit.tests/unit.tests./A", + data={ + "ttl": 300, + "resource_records": [ + {"content": ["1.2.3.4", "1.2.3.5"]} + ], + }, + ), + ] + ) + # expected number of total calls + self.assertEquals(7, provider._client._request.call_count) + + provider._client._request.reset_mock() + + # delete 1 and update 1 + provider._client.zone_records = Mock( + return_value=[ + { + "name": "www", + "ttl": 300, + "type": "A", + "resource_records": [{"content": ["1.2.3.4"]}], + }, + { + "name": "ttl", + "ttl": 600, + "type": "A", + "resource_records": [{"content": ["3.2.3.4"]}], + }, + ] + ) + + # Domain exists, we don't care about return + resp.json.side_effect = ["{}"] + + wanted = Zone("unit.tests.", []) + wanted.add_record( + Record.new( + wanted, "ttl", {"ttl": 300, "type": "A", "value": "3.2.3.4"} + ) + ) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEquals(2, len(plan.changes)) + self.assertEquals(2, provider.apply(plan)) + + provider._client._request.assert_has_calls( + [ + call("DELETE", "/zones/unit.tests/www.unit.tests./A"), + call( + "PUT", + "/zones/unit.tests/ttl.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["3.2.3.4"]}], + }, + ), + ] + ) From 88b003130e423896f90cf3767b2b263c4a13a6e7 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 9 Mar 2021 11:28:09 +0300 Subject: [PATCH 02/14] fix API incorrect behaviour while returning records data: API always must return type in upper case, but at this moment it returns record type as it was provided on record creation --- octodns/provider/gcore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 42e2cb7..5865431 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -141,7 +141,7 @@ class GCoreProvider(BaseProvider): values = defaultdict(defaultdict) records, exists = self.zone_records(zone) for record in records: - _type = record["type"] + _type = record["type"].upper() if _type not in self.SUPPORTS: continue rr_name = record["name"].replace(zone.name, "") From 2f90ce475652a32c67663e99f2737d31ca911dfd Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 9 Mar 2021 16:26:22 +0300 Subject: [PATCH 03/14] add more information in logs when RR create/update/delete fail --- octodns/provider/gcore.py | 46 ++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 5865431..6387771 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -179,27 +179,39 @@ class GCoreProvider(BaseProvider): _params_for_AAAA = _params_for_single def _apply_create(self, change): - new = change.new - rrset_name = self._build_rrset_name(new) - data = getattr(self, "_params_for_{}".format(new._type))(new) - self._client.record_create( - new.zone.name[:-1], rrset_name, new._type, data - ) + try: + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_create( + new.zone.name[:-1], rrset_name, new._type, data + ) + except: + self.log.exception("failed to create RR: %s", change) + rai def _apply_update(self, change): - new = change.new - rrset_name = self._build_rrset_name(new) - data = getattr(self, "_params_for_{}".format(new._type))(new) - self._client.record_update( - new.zone.name[:-1], rrset_name, new._type, data - ) + try: + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_update( + new.zone.name[:-1], rrset_name, new._type, data + ) + except: + self.log.exception("failed to update RR: %s", change) + raise def _apply_delete(self, change): - existing = change.existing - rrset_name = self._build_rrset_name(existing) - self._client.record_delete( - existing.zone.name[:-1], rrset_name, existing._type - ) + try: + existing = change.existing + rrset_name = self._build_rrset_name(existing) + self._client.record_delete( + existing.zone.name[:-1], rrset_name, existing._type + ) + except: + self.log.exception("failed to delete RR: %s", change) + raise def _apply(self, plan): desired = plan.desired From 04a9b2b9b869697e32e48472584feafde4cb2d6a Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Fri, 12 Mar 2021 15:00:24 +0300 Subject: [PATCH 04/14] add logging for all apply actions, instead of in case of error(s) --- octodns/provider/gcore.py | 49 ++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 6387771..28c9fad 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -179,39 +179,30 @@ class GCoreProvider(BaseProvider): _params_for_AAAA = _params_for_single def _apply_create(self, change): - try: - new = change.new - rrset_name = self._build_rrset_name(new) - data = getattr(self, "_params_for_{}".format(new._type))(new) - self._client.record_create( - new.zone.name[:-1], rrset_name, new._type, data - ) - except: - self.log.exception("failed to create RR: %s", change) - rai + self.log.info("creating: %s", change) + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_create( + new.zone.name[:-1], rrset_name, new._type, data + ) def _apply_update(self, change): - try: - new = change.new - rrset_name = self._build_rrset_name(new) - data = getattr(self, "_params_for_{}".format(new._type))(new) - self._client.record_update( - new.zone.name[:-1], rrset_name, new._type, data - ) - except: - self.log.exception("failed to update RR: %s", change) - raise + self.log.info("updating: %s", change) + new = change.new + rrset_name = self._build_rrset_name(new) + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_update( + new.zone.name[:-1], rrset_name, new._type, data + ) def _apply_delete(self, change): - try: - existing = change.existing - rrset_name = self._build_rrset_name(existing) - self._client.record_delete( - existing.zone.name[:-1], rrset_name, existing._type - ) - except: - self.log.exception("failed to delete RR: %s", change) - raise + self.log.info("deleting: %s", change) + existing = change.existing + rrset_name = self._build_rrset_name(existing) + self._client.record_delete( + existing.zone.name[:-1], rrset_name, existing._type + ) def _apply(self, plan): desired = plan.desired From ac56a36f432ab8a071d627828d1613b3e31a15a7 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Mon, 22 Mar 2021 20:57:10 +0300 Subject: [PATCH 05/14] multiple resource records values should be sent as separate 'content'-s --- octodns/provider/gcore.py | 4 +++- tests/test_octodns_provider_gcore.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 28c9fad..246a77a 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -172,7 +172,9 @@ class GCoreProvider(BaseProvider): def _params_for_single(self, record): return { "ttl": record.ttl, - "resource_records": [{"content": record.values}], + "resource_records": [ + {"content": [value]} for value in record.values + ], } _params_for_A = _params_for_single diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 74762da..5671e15 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -192,7 +192,8 @@ class TestGCoreProvider(TestCase): data={ "ttl": 300, "resource_records": [ - {"content": ["1.2.3.4", "1.2.3.5"]} + {"content": ["1.2.3.4"]}, + {"content": ["1.2.3.5"]}, ], }, ), From 70555ed546ce31d8cb72588ca2c66acedbdda43e Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 23 Mar 2021 11:35:12 +0300 Subject: [PATCH 06/14] add new API parameter has been added to fetch all resource records --- octodns/provider/gcore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 246a77a..60e8bad 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -68,7 +68,7 @@ class GCoreClient(object): def zone_records(self, zone_name): rrsets = self._request( - "GET", "{}/{}/rrsets".format(self.ROOT_ZONES, zone_name) + "GET", "{}/{}/rrsets?all=true".format(self.ROOT_ZONES, zone_name) ).json() records = rrsets["rrsets"] return records From 4a1e11c3942ef526239b06673004293dbb8d6b3b Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 23 Mar 2021 11:48:23 +0300 Subject: [PATCH 07/14] commit missing changes for tests --- tests/test_octodns_provider_gcore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 5671e15..785e5e9 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -153,7 +153,7 @@ class TestGCoreProvider(TestCase): provider._client._request.assert_has_calls( [ - call("GET", "/zones/unit.tests/rrsets"), + call("GET", "/zones/unit.tests/rrsets?all=true"), call("GET", "/zones/unit.tests"), call("POST", "/zones", data={"name": "unit.tests"}), call( From 988e8d27fb61ab86a7e8f565c53cf3ff9fae3de7 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Fri, 9 Apr 2021 11:45:40 +0300 Subject: [PATCH 08/14] fix populate for multiple resource records in single rrset --- octodns/provider/gcore.py | 20 +++++++++++++++++--- tests/fixtures/gcore-no-changes.json | 5 ++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 60e8bad..5582dd8 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -36,9 +36,10 @@ class GCoreClient(object): ROOT_ZONES = "/zones" - def __init__(self, base_url, token): + def __init__(self, log, base_url, token): session = Session() session.headers.update({"Authorization": "Bearer {}".format(token)}) + self.log = log self._session = session self._base_url = base_url @@ -48,10 +49,19 @@ class GCoreClient(object): method, url, params=params, json=data, timeout=30.0 ) if r.status_code == 400: + self.log.error( + "bad request %r has been sent to %r: %s", data, url, r.text + ) raise GCoreClientBadRequest(r) elif r.status_code == 404: + self.log.error( + "resource %r not found: %s", url, r.text + ) raise GCoreClientNotFound(r) elif r.status_code == 500: + self.log.error( + "server error no %r to %r: %s", data, url, r.text + ) raise GCoreClientException(r) r.raise_for_status() return r @@ -112,13 +122,17 @@ class GCoreProvider(BaseProvider): self.log = logging.getLogger("GCoreProvider[{}]".format(id)) self.log.debug("__init__: id=%s, token=***", id) super(GCoreProvider, self).__init__(id, *args, **kwargs) - self._client = GCoreClient(base_url, token) + self._client = GCoreClient(self.log, base_url, token) def _data_for_single(self, _type, record): return { "ttl": record["ttl"], "type": _type, - "values": record["resource_records"][0]["content"], + "values": [ + rr_value + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ], } _data_for_A = _data_for_single diff --git a/tests/fixtures/gcore-no-changes.json b/tests/fixtures/gcore-no-changes.json index ad62c94..a70fb82 100644 --- a/tests/fixtures/gcore-no-changes.json +++ b/tests/fixtures/gcore-no-changes.json @@ -5,7 +5,10 @@ "ttl": 300, "resource_records": [{ "content": [ - "1.2.3.4", + "1.2.3.4" + ] + }, { + "content": [ "1.2.3.5" ] }] From 0ff933244a4bb2d4777a74fd3af39c00a6b19125 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 20 Apr 2021 14:33:42 +0300 Subject: [PATCH 09/14] allow to pass login/password which will be used to acquire token for further usage --- octodns/provider/gcore.py | 104 ++++++++++++++++++++------- tests/test_octodns_provider_gcore.py | 72 +++++++++++++++---- 2 files changed, 137 insertions(+), 39 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 5582dd8..a52439f 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -11,7 +11,9 @@ from __future__ import ( from collections import defaultdict from requests import Session +import http import logging +import urllib.parse from ..record import Record from .base import BaseProvider @@ -34,51 +36,82 @@ class GCoreClientNotFound(GCoreClientException): class GCoreClient(object): - ROOT_ZONES = "/zones" + ROOT_ZONES = "zones" - def __init__(self, log, base_url, token): - session = Session() - session.headers.update({"Authorization": "Bearer {}".format(token)}) + def __init__( + self, + log, + api_url, + auth_url, + token=None, + login=None, + password=None, + ): self.log = log - self._session = session - self._base_url = base_url + self._session = Session() + self._api_url = api_url + if token is not None: + self._session.headers.update( + {"Authorization": "APIKey {}".format(token)} + ) + elif login is not None and password is not None: + token = self._auth(auth_url, login, password) + self._session.headers.update( + {"Authorization": "Bearer {}".format(token)} + ) + else: + raise ValueError("either token or login & password must be set") - def _request(self, method, path, params={}, data=None): - url = "{}{}".format(self._base_url, path) + def _auth(self, url, login, password): + # well, can't use _request, since API returns 400 if credentials + # invalid which will be logged, but we don't want do this + r = self._session.request( + "POST", + self._build_url(url, "auth", "jwt", "login"), + json={"username": login, "password": password}, + ) + r.raise_for_status() + return r.json()["access"] + + def _request(self, method, url, params=None, data=None): r = self._session.request( method, url, params=params, json=data, timeout=30.0 ) - if r.status_code == 400: + if r.status_code == http.HTTPStatus.BAD_REQUEST: self.log.error( "bad request %r has been sent to %r: %s", data, url, r.text ) raise GCoreClientBadRequest(r) - elif r.status_code == 404: - self.log.error( - "resource %r not found: %s", url, r.text - ) + elif r.status_code == http.HTTPStatus.NOT_FOUND: + self.log.error("resource %r not found: %s", url, r.text) raise GCoreClientNotFound(r) - elif r.status_code == 500: - self.log.error( - "server error no %r to %r: %s", data, url, r.text - ) + elif r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR: + self.log.error("server error no %r to %r: %s", data, url, r.text) raise GCoreClientException(r) r.raise_for_status() return r def zone(self, zone_name): return self._request( - "GET", "{}/{}".format(self.ROOT_ZONES, zone_name) + "GET", self._build_url(self._api_url, self.ROOT_ZONES, zone_name) ).json() def zone_create(self, zone_name): return self._request( - "POST", self.ROOT_ZONES, data={"name": zone_name} + "POST", + self._build_url(self._api_url, self.ROOT_ZONES), + data={"name": zone_name}, ).json() def zone_records(self, zone_name): rrsets = self._request( - "GET", "{}/{}/rrsets?all=true".format(self.ROOT_ZONES, zone_name) + "GET", + "{}".format( + self._build_url( + self._api_url, self.ROOT_ZONES, zone_name, "rrsets" + ) + ), + params={"all": "true"}, ).json() records = rrsets["rrsets"] return records @@ -97,10 +130,17 @@ class GCoreClient(object): self._request("DELETE", self._rrset_url(zone_name, rrset_name, type_)) def _rrset_url(self, zone_name, rrset_name, type_): - return "{}/{}/{}/{}".format( - self.ROOT_ZONES, zone_name, rrset_name, type_ + return self._build_url( + self._api_url, self.ROOT_ZONES, zone_name, rrset_name, type_ ) + @staticmethod + def _build_url(base, *items): + for i in items: + base = base.strip("/") + "/" + base = urllib.parse.urljoin(base, i) + return base + class GCoreProvider(BaseProvider): """ @@ -108,8 +148,12 @@ class GCoreProvider(BaseProvider): gcore: class: octodns.provider.gcore.GCoreProvider - # Your API key (required) + # Your API key token: XXXXXXXXXXXX + # or login + password + login: XXXXXXXXXXXX + password: XXXXXXXXXXXX + # auth_url: https://api.gcdn.co # url: https://dnsapi.gcorelabs.com/v2 """ @@ -117,12 +161,18 @@ class GCoreProvider(BaseProvider): SUPPORTS_DYNAMIC = False SUPPORTS = set(("A", "AAAA")) - def __init__(self, id, token, *args, **kwargs): - base_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") + def __init__(self, id, *args, **kwargs): + token = kwargs.pop("token", None) + login = kwargs.pop("login", None) + password = kwargs.pop("password", None) + api_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") + auth_url = kwargs.pop("auth_url", "https://api.gcdn.co") self.log = logging.getLogger("GCoreProvider[{}]".format(id)) - self.log.debug("__init__: id=%s, token=***", id) + self.log.debug("__init__: id=%s", id) super(GCoreProvider, self).__init__(id, *args, **kwargs) - self._client = GCoreClient(self.log, base_url, token) + self._client = GCoreClient( + self.log, api_url, auth_url, token, login, password + ) def _data_for_single(self, _type, record): return { diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 785e5e9..aa536f1 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -33,7 +33,7 @@ class TestGCoreProvider(TestCase): def test_populate(self): - provider = GCoreProvider("test_id", "token") + provider = GCoreProvider("test_id", token="token") # 400 - Bad Request. with requests_mock() as mock: @@ -52,7 +52,7 @@ class TestGCoreProvider(TestCase): with self.assertRaises(GCoreClientNotFound) as ctx: zone = Zone("unit.tests.", []) - provider._client.zone(zone) + provider._client.zone(zone.name) self.assertIn( '"error":"zone is not found"', text_type(ctx.exception) ) @@ -66,6 +66,48 @@ class TestGCoreProvider(TestCase): provider.populate(zone) self.assertEquals("Things caught fire", text_type(ctx.exception)) + # No credentials or token error + with requests_mock() as mock: + with self.assertRaises(ValueError) as ctx: + GCoreProvider("test_id") + self.assertEquals( + "either token or login & password must be set", + text_type(ctx.exception), + ) + + # Auth with login password + with requests_mock() as mock: + + def match_body(request): + return {"username": "foo", "password": "bar"} == request.json() + + auth_url = "http://api/auth/jwt/login" + mock.post( + auth_url, + additional_matcher=match_body, + status_code=200, + json={"access": "access"}, + ) + + providerPassword = GCoreProvider( + "test_id", + url="http://dns", + auth_url="http://api", + login="foo", + password="bar", + ) + assert mock.called + + # make sure token passed in header + zone_rrset_url = "http://dns/zones/unit.tests/rrsets?all=true" + mock.get( + zone_rrset_url, + request_headers={"Authorization": "Bearer access"}, + status_code=404, + ) + zone = Zone("unit.tests.", []) + assert not providerPassword.populate(zone) + # No diffs == no changes with requests_mock() as mock: base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" @@ -97,7 +139,7 @@ class TestGCoreProvider(TestCase): ) def test_apply(self): - provider = GCoreProvider("test_id", "token") + provider = GCoreProvider("test_id", url="http://api", token="token") # Zone does not exists but can be created. with requests_mock() as mock: @@ -153,12 +195,16 @@ class TestGCoreProvider(TestCase): provider._client._request.assert_has_calls( [ - call("GET", "/zones/unit.tests/rrsets?all=true"), - call("GET", "/zones/unit.tests"), - call("POST", "/zones", data={"name": "unit.tests"}), + call( + "GET", + "http://api/zones/unit.tests/rrsets", + params={"all": "true"}, + ), + call("GET", "http://api/zones/unit.tests"), + call("POST", "http://api/zones", data={"name": "unit.tests"}), call( "POST", - "/zones/unit.tests/www.sub.unit.tests./A", + "http://api/zones/unit.tests/www.sub.unit.tests./A", data={ "ttl": 300, "resource_records": [{"content": ["2.2.3.6"]}], @@ -166,7 +212,7 @@ class TestGCoreProvider(TestCase): ), call( "POST", - "/zones/unit.tests/www.unit.tests./A", + "http://api/zones/unit.tests/www.unit.tests./A", data={ "ttl": 300, "resource_records": [{"content": ["2.2.3.6"]}], @@ -174,7 +220,7 @@ class TestGCoreProvider(TestCase): ), call( "POST", - "/zones/unit.tests/aaaa.unit.tests./AAAA", + "http://api/zones/unit.tests/aaaa.unit.tests./AAAA", data={ "ttl": 600, "resource_records": [ @@ -188,7 +234,7 @@ class TestGCoreProvider(TestCase): ), call( "POST", - "/zones/unit.tests/unit.tests./A", + "http://api/zones/unit.tests/unit.tests./A", data={ "ttl": 300, "resource_records": [ @@ -239,10 +285,12 @@ class TestGCoreProvider(TestCase): provider._client._request.assert_has_calls( [ - call("DELETE", "/zones/unit.tests/www.unit.tests./A"), + call( + "DELETE", "http://api/zones/unit.tests/www.unit.tests./A" + ), call( "PUT", - "/zones/unit.tests/ttl.unit.tests./A", + "http://api/zones/unit.tests/ttl.unit.tests./A", data={ "ttl": 300, "resource_records": [{"content": ["3.2.3.4"]}], From 04d87fdf35ed4cc9e57c462f8721acf3f14d7910 Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 20 Apr 2021 16:37:09 +0300 Subject: [PATCH 10/14] allow to customize token_type, since there are two types of tokens: permanent (APIKey) and JWT (Bearer) --- octodns/provider/gcore.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index a52439f..041e19f 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -44,15 +44,16 @@ class GCoreClient(object): api_url, auth_url, token=None, + token_type=None, login=None, password=None, ): self.log = log self._session = Session() self._api_url = api_url - if token is not None: + if token is not None and token_type is not None: self._session.headers.update( - {"Authorization": "APIKey {}".format(token)} + {"Authorization": "{} {}".format(token_type, token)} ) elif login is not None and password is not None: token = self._auth(auth_url, login, password) @@ -150,6 +151,7 @@ class GCoreProvider(BaseProvider): class: octodns.provider.gcore.GCoreProvider # Your API key token: XXXXXXXXXXXX + # token_type: APIKey # or login + password login: XXXXXXXXXXXX password: XXXXXXXXXXXX @@ -163,6 +165,7 @@ class GCoreProvider(BaseProvider): def __init__(self, id, *args, **kwargs): token = kwargs.pop("token", None) + token_type = kwargs.pop("token_type", "APIKey") login = kwargs.pop("login", None) password = kwargs.pop("password", None) api_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") @@ -171,7 +174,13 @@ class GCoreProvider(BaseProvider): self.log.debug("__init__: id=%s", id) super(GCoreProvider, self).__init__(id, *args, **kwargs) self._client = GCoreClient( - self.log, api_url, auth_url, token, login, password + self.log, + api_url, + auth_url, + token=token, + token_type=token_type, + login=login, + password=password, ) def _data_for_single(self, _type, record): From 6ac368c488f41857b5fe4ac1a686d4de4db8fe5f Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Mon, 24 May 2021 15:38:31 +0300 Subject: [PATCH 11/14] rrset names may include ending dot, may not since new API update --- octodns/provider/gcore.py | 19 ++++--------------- tests/fixtures/gcore-no-changes.json | 6 +++--- tests/fixtures/gcore-records.json | 4 ++-- tests/fixtures/gcore-zone.json | 2 +- tests/test_octodns_provider_gcore.py | 3 +++ 5 files changed, 13 insertions(+), 21 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 041e19f..d5c7fa0 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -217,9 +217,7 @@ class GCoreProvider(BaseProvider): _type = record["type"].upper() if _type not in self.SUPPORTS: continue - rr_name = record["name"].replace(zone.name, "") - if len(rr_name) > 0 and rr_name.endswith("."): - rr_name = rr_name[:-1] + rr_name = zone.hostname_from_fqdn(record["name"]) values[rr_name][_type] = record before = len(zone.records) @@ -256,27 +254,24 @@ class GCoreProvider(BaseProvider): def _apply_create(self, change): self.log.info("creating: %s", change) new = change.new - rrset_name = self._build_rrset_name(new) data = getattr(self, "_params_for_{}".format(new._type))(new) self._client.record_create( - new.zone.name[:-1], rrset_name, new._type, data + new.zone.name[:-1], new.fqdn, new._type, data ) def _apply_update(self, change): self.log.info("updating: %s", change) new = change.new - rrset_name = self._build_rrset_name(new) data = getattr(self, "_params_for_{}".format(new._type))(new) self._client.record_update( - new.zone.name[:-1], rrset_name, new._type, data + new.zone.name[:-1], new.fqdn, new._type, data ) def _apply_delete(self, change): self.log.info("deleting: %s", change) existing = change.existing - rrset_name = self._build_rrset_name(existing) self._client.record_delete( - existing.zone.name[:-1], rrset_name, existing._type + existing.zone.name[:-1], existing.fqdn, existing._type ) def _apply(self, plan): @@ -299,9 +294,3 @@ class GCoreProvider(BaseProvider): for change in changes: class_name = change.__class__.__name__ getattr(self, "_apply_{}".format(class_name.lower()))(change) - - @staticmethod - def _build_rrset_name(record): - if len(record.name) > 0: - return "{}.{}".format(record.name, record.zone.name) - return record.zone.name diff --git a/tests/fixtures/gcore-no-changes.json b/tests/fixtures/gcore-no-changes.json index a70fb82..e5ff8c9 100644 --- a/tests/fixtures/gcore-no-changes.json +++ b/tests/fixtures/gcore-no-changes.json @@ -1,6 +1,6 @@ { "rrsets": [{ - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "ttl": 300, "resource_records": [{ @@ -13,7 +13,7 @@ ] }] }, { - "name": "aaaa.unit.tests.", + "name": "aaaa.unit.tests", "type": "AAAA", "ttl": 600, "resource_records": [{ @@ -40,7 +40,7 @@ ] }] }, { - "name": "unit.tests.", + "name": "unit.tests", "type": "ns", "ttl": 300, "resource_records": [{ diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json index 4d4685e..4086049 100644 --- a/tests/fixtures/gcore-records.json +++ b/tests/fixtures/gcore-records.json @@ -1,6 +1,6 @@ { "rrsets": [{ - "name": "unit.tests.", + "name": "unit.tests", "type": "A", "ttl": 300, "resource_records": [{ @@ -9,7 +9,7 @@ ] }] }, { - "name": "unit.tests.", + "name": "unit.tests", "type": "ns", "ttl": 300, "resource_records": [{ diff --git a/tests/fixtures/gcore-zone.json b/tests/fixtures/gcore-zone.json index 7f70275..925af72 100644 --- a/tests/fixtures/gcore-zone.json +++ b/tests/fixtures/gcore-zone.json @@ -11,7 +11,7 @@ "records": [ { "id": 12419, - "name": "unit.test.", + "name": "unit.test", "type": "ns", "ttl": 300, "short_answers": [ diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index aa536f1..06e199e 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -117,6 +117,9 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) provider.populate(zone) self.assertEquals(4, len(zone.records)) + self.assertEquals( + {"aaaa", "www", "www.sub", ""}, {r.name for r in zone.records} + ) changes = self.expected.changes(zone, provider) self.assertEquals(0, len(changes)) From c6486ce3d1ac29ff706ebfb95a9a4c3fbe78b8be Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Mon, 26 Jul 2021 17:57:16 +0300 Subject: [PATCH 12/14] add support for NS, MX, TXT, SRV, CNAME, PTR --- README.md | 2 +- octodns/provider/gcore.py | 117 ++++++++++- tests/fixtures/gcore-no-changes.json | 285 ++++++++++++++++++++++----- tests/fixtures/gcore-records.json | 221 +++++++++++++++++++-- tests/test_octodns_provider_gcore.py | 145 ++++++++++++-- 5 files changed, 677 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index 52d3a25..2323633 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | -| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA | No | | +| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA, NS, MX, TXT, SRV, CNAME, PTR | No | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | | [Ns1Provider](/octodns/provider/ns1.py) | ns1-python | All | Yes | Missing `NA` geo target | diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index d5c7fa0..c551f03 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -161,7 +161,7 @@ class GCoreProvider(BaseProvider): SUPPORTS_GEO = False SUPPORTS_DYNAMIC = False - SUPPORTS = set(("A", "AAAA")) + SUPPORTS = set(("A", "AAAA", "NS", "MX", "TXT", "SRV", "CNAME", "PTR")) def __init__(self, id, *args, **kwargs): token = kwargs.pop("token", None) @@ -183,7 +183,22 @@ class GCoreProvider(BaseProvider): password=password, ) + def _add_dot_if_need(self, value): + return "{}.".format(value) if not value.endswith(".") else value + def _data_for_single(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "value": self._add_dot_if_need( + record["resource_records"][0]["content"][0] + ), + } + + _data_for_CNAME = _data_for_single + _data_for_PTR = _data_for_single + + def _data_for_multiple(self, _type, record): return { "ttl": record["ttl"], "type": _type, @@ -194,8 +209,62 @@ class GCoreProvider(BaseProvider): ], } - _data_for_A = _data_for_single - _data_for_AAAA = _data_for_single + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_TXT(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + rr_value.replace(";", "\\;") + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ], + } + + def _data_for_MX(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + dict( + preference=preference, + exchange=self._add_dot_if_need(exchange), + ) + for preference, exchange in map( + lambda x: x["content"], record["resource_records"] + ) + ], + } + + def _data_for_NS(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + self._add_dot_if_need(rr_value) + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ], + } + + def _data_for_SRV(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + dict( + priority=priority, + weight=weight, + port=port, + target=self._add_dot_if_need(target), + ) + for priority, weight, port, target in map( + lambda x: x["content"], record["resource_records"] + ) + ], + } def zone_records(self, zone): try: @@ -241,6 +310,15 @@ class GCoreProvider(BaseProvider): return exists def _params_for_single(self, record): + return { + "ttl": record.ttl, + "resource_records": [{"content": [record.value]}], + } + + _params_for_CNAME = _params_for_single + _params_for_PTR = _params_for_single + + def _params_for_multiple(self, record): return { "ttl": record.ttl, "resource_records": [ @@ -248,8 +326,37 @@ class GCoreProvider(BaseProvider): ], } - _params_for_A = _params_for_single - _params_for_AAAA = _params_for_single + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + _params_for_NS = _params_for_multiple + + def _params_for_TXT(self, record): + # print(record.values) + return { + "ttl": record.ttl, + "resource_records": [ + {"content": [value.replace("\\;", ";")]} + for value in record.values + ], + } + + def _params_for_MX(self, record): + return { + "ttl": record.ttl, + "resource_records": [ + {"content": [rec.preference, rec.exchange]} + for rec in record.values + ], + } + + def _params_for_SRV(self, record): + return { + "ttl": record.ttl, + "resource_records": [ + {"content": [rec.priority, rec.weight, rec.port, rec.target]} + for rec in record.values + ], + } def _apply_create(self, change): self.log.info("creating: %s", change) diff --git a/tests/fixtures/gcore-no-changes.json b/tests/fixtures/gcore-no-changes.json index e5ff8c9..b1a3b25 100644 --- a/tests/fixtures/gcore-no-changes.json +++ b/tests/fixtures/gcore-no-changes.json @@ -1,56 +1,245 @@ { - "rrsets": [{ - "name": "unit.tests", - "type": "A", - "ttl": 300, - "resource_records": [{ - "content": [ - "1.2.3.4" + "rrsets": [ + { + "name": "unit.tests", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "1.2.3.4" + ] + }, + { + "content": [ + "1.2.3.5" + ] + } ] - }, { - "content": [ - "1.2.3.5" + }, + { + "name": "unit.tests", + "type": "NS", + "ttl": 300, + "resource_records": [ + { + "content": [ + "ns2.gcdn.services" + ] + }, + { + "content": [ + "ns1.gcorelabs.net" + ] + } ] - }] - }, { - "name": "aaaa.unit.tests", - "type": "AAAA", - "ttl": 600, - "resource_records": [{ - "content": [ - "2601:644:500:e210:62f8:1dff:feb8:947a" + }, + { + "name": "_imap._tcp", + "type": "SRV", + "ttl": 600, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } ] - }] - }, { - "name": "www.unit.tests.", - "type": "A", - "ttl": 300, - "resource_records": [{ - "content": [ - "2.2.3.6" + }, + { + "name": "_pop3._tcp", + "type": "SRV", + "ttl": 600, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } ] - }] - }, { - "name": "www.sub.unit.tests.", - "type": "A", - "ttl": 300, - "resource_records": [{ - "content": [ - "2.2.3.6" + }, + { + "name": "_srv._tcp", + "type": "SRV", + "ttl": 600, + "resource_records": [ + { + "content": [ + 12, + 20, + 30, + "foo-2.unit.tests" + ] + }, + { + "content": [ + 10, + 20, + 30, + "foo-1.unit.tests" + ] + } ] - }] - }, { - "name": "unit.tests", - "type": "ns", - "ttl": 300, - "resource_records": [{ - "content": [ - "ns2.gcdn.services" + }, + { + "name": "aaaa.unit.tests", + "type": "AAAA", + "ttl": 600, + "resource_records": [ + { + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + } ] - }, { - "content": [ - "ns1.gcorelabs.net" + }, + { + "name": "cname.unit.tests", + "type": "CNAME", + "ttl": 300, + "resource_records": [ + { + "content": [ + "unit.tests." + ] + } ] - }] - }] -} + }, + { + "name": "excluded.unit.tests", + "type": "CNAME", + "ttl": 3600, + "resource_records": [ + { + "content": [ + "unit.tests." + ] + } + ] + }, + { + "name": "mx.unit.tests", + "type": "MX", + "ttl": 300, + "resource_records": [ + { + "content": [ + 40, + "smtp-1.unit.tests." + ] + }, + { + "content": [ + 20, + "smtp-2.unit.tests." + ] + }, + { + "content": [ + 30, + "smtp-3.unit.tests." + ] + }, + { + "content": [ + 10, + "smtp-4.unit.tests." + ] + } + ] + }, + { + "name": "ptr.unit.tests.", + "type": "PTR", + "ttl": 300, + "resource_records": [ + { + "content": [ + "foo.bar.com" + ] + } + ] + }, + { + "name": "sub.unit.tests", + "type": "NS", + "ttl": 3600, + "resource_records": [ + { + "content": [ + "6.2.3.4" + ] + }, + { + "content": [ + "7.2.3.4" + ] + } + ] + }, + { + "name": "txt.unit.tests", + "type": "TXT", + "ttl": 600, + "resource_records": [ + { + "content": [ + "Bah bah black sheep" + ] + }, + { + "content": [ + "have you any wool." + ] + }, + { + "content": [ + "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs" + ] + } + ] + }, + { + "name": "www.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + }, + { + "name": "www.sub.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + }, + { + "name": "spf.sub.unit.tests.", + "type": "SPF", + "ttl": 600, + "resource_records": [ + { + "content": [ + "v=spf1 ip4:192.168.0.1/16-all" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json index 4086049..ba29246 100644 --- a/tests/fixtures/gcore-records.json +++ b/tests/fixtures/gcore-records.json @@ -1,25 +1,204 @@ { - "rrsets": [{ - "name": "unit.tests", - "type": "A", - "ttl": 300, - "resource_records": [{ - "content": [ - "1.2.3.4" + "rrsets": [ + { + "name": "unit.tests", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "1.2.3.4" + ] + } ] - }] - }, { - "name": "unit.tests", - "type": "ns", - "ttl": 300, - "resource_records": [{ - "content": [ - "ns2.gcdn.services" + }, + { + "name": "unit.tests", + "type": "NS", + "ttl": 300, + "resource_records": [ + { + "content": [ + "ns2.gcdn.services" + ] + }, + { + "content": [ + "ns1.gcorelabs.net" + ] + } ] - }, { - "content": [ - "ns1.gcorelabs.net" + }, + { + "name": "_imap._tcp", + "type": "SRV", + "ttl": 1200, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } ] - }] - }] -} + }, + { + "name": "_pop3._tcp", + "type": "SRV", + "ttl": 1200, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } + ] + }, + { + "name": "_srv._tcp", + "type": "SRV", + "ttl": 1200, + "resource_records": [ + { + "content": [ + 12, + 20, + 30, + "foo-2.unit.tests." + ] + }, + { + "content": [ + 10, + 20, + 30, + "foo-1.unit.tests." + ] + } + ] + }, + { + "name": "aaaa.unit.tests", + "type": "AAAA", + "ttl": 600, + "resource_records": [ + { + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + } + ] + }, + { + "name": "cname.unit.tests", + "type": "CNAME", + "ttl": 300, + "resource_records": [ + { + "content": [ + "unit.tests." + ] + } + ] + }, + { + "name": "mx.unit.tests", + "type": "MX", + "ttl": 600, + "resource_records": [ + { + "content": [ + 40, + "smtp-1.unit.tests." + ] + }, + { + "content": [ + 20, + "smtp-2.unit.tests." + ] + } + ] + }, + { + "name": "ptr.unit.tests.", + "type": "PTR", + "ttl": 300, + "resource_records": [ + { + "content": [ + "foo.bar.com" + ] + } + ] + }, + { + "name": "sub.unit.tests", + "type": "NS", + "ttl": 300, + "resource_records": [ + { + "content": [ + "6.2.3.4" + ] + }, + { + "content": [ + "7.2.3.4" + ] + } + ] + }, + { + "name": "txt.unit.tests", + "type": "TXT", + "ttl": 300, + "resource_records": [ + { + "content": [ + "\"Bah bah black sheep\"" + ] + }, + { + "content": [ + "\"have you any wool.\"" + ] + }, + { + "content": [ + "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"" + ] + } + ] + }, + { + "name": "www.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + }, + { + "name": "www.sub.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 06e199e..2ddbb59 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -64,13 +64,13 @@ class TestGCoreProvider(TestCase): with self.assertRaises(GCoreClientException) as ctx: zone = Zone("unit.tests.", []) provider.populate(zone) - self.assertEquals("Things caught fire", text_type(ctx.exception)) + self.assertEqual("Things caught fire", text_type(ctx.exception)) # No credentials or token error with requests_mock() as mock: with self.assertRaises(ValueError) as ctx: GCoreProvider("test_id") - self.assertEquals( + self.assertEqual( "either token or login & password must be set", text_type(ctx.exception), ) @@ -116,14 +116,29 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) provider.populate(zone) - self.assertEquals(4, len(zone.records)) - self.assertEquals( - {"aaaa", "www", "www.sub", ""}, {r.name for r in zone.records} + self.assertEqual(14, len(zone.records)) + self.assertEqual( + { + "", + "_imap._tcp", + "_pop3._tcp", + "_srv._tcp", + "aaaa", + "cname", + "excluded", + "mx", + "ptr", + "sub", + "txt", + "www", + "www.sub", + }, + {r.name for r in zone.records}, ) changes = self.expected.changes(zone, provider) - self.assertEquals(0, len(changes)) + self.assertEqual(0, len(changes)) - # 3 removed + 1 modified + # 1 removed + 7 modified with requests_mock() as mock: base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" with open("tests/fixtures/gcore-records.json") as fh: @@ -131,14 +146,14 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) provider.populate(zone) - self.assertEquals(1, len(zone.records)) + self.assertEqual(13, len(zone.records)) changes = self.expected.changes(zone, provider) - self.assertEquals(4, len(changes)) - self.assertEquals( - 3, len([c for c in changes if isinstance(c, Delete)]) + self.assertEqual(8, len(changes)) + self.assertEqual( + 1, len([c for c in changes if isinstance(c, Delete)]) ) - self.assertEquals( - 1, len([c for c in changes if isinstance(c, Update)]) + self.assertEqual( + 7, len([c for c in changes if isinstance(c, Update)]) ) def test_apply(self): @@ -192,8 +207,8 @@ class TestGCoreProvider(TestCase): plan = provider.plan(self.expected) # create all - self.assertEquals(4, len(plan.changes)) - self.assertEquals(4, provider.apply(plan)) + self.assertEqual(13, len(plan.changes)) + self.assertEqual(13, provider.apply(plan)) self.assertFalse(plan.exists) provider._client._request.assert_has_calls( @@ -221,6 +236,73 @@ class TestGCoreProvider(TestCase): "resource_records": [{"content": ["2.2.3.6"]}], }, ), + call( + "POST", + "http://api/zones/unit.tests/txt.unit.tests./TXT", + data={ + "ttl": 600, + "resource_records": [ + {"content": ["Bah bah black sheep"]}, + {"content": ["have you any wool."]}, + { + "content": [ + "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+" + "of/long/string+with+numb3rs" + ] + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/sub.unit.tests./NS", + data={ + "ttl": 3600, + "resource_records": [ + {"content": ["6.2.3.4."]}, + {"content": ["7.2.3.4."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/ptr.unit.tests./PTR", + data={ + "ttl": 300, + "resource_records": [ + {"content": ["foo.bar.com."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/mx.unit.tests./MX", + data={ + "ttl": 300, + "resource_records": [ + {"content": [10, "smtp-4.unit.tests."]}, + {"content": [20, "smtp-2.unit.tests."]}, + {"content": [30, "smtp-3.unit.tests."]}, + {"content": [40, "smtp-1.unit.tests."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/excluded.unit.tests./CNAME", + data={ + "ttl": 3600, + "resource_records": [{"content": ["unit.tests."]}], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/cname.unit.tests./CNAME", + data={ + "ttl": 300, + "resource_records": [{"content": ["unit.tests."]}], + }, + ), call( "POST", "http://api/zones/unit.tests/aaaa.unit.tests./AAAA", @@ -235,6 +317,33 @@ class TestGCoreProvider(TestCase): ], }, ), + call( + "POST", + "http://api/zones/unit.tests/_srv._tcp.unit.tests./SRV", + data={ + "ttl": 600, + "resource_records": [ + {"content": [10, 20, 30, "foo-1.unit.tests."]}, + {"content": [12, 20, 30, "foo-2.unit.tests."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/_pop3._tcp.unit.tests./SRV", + data={ + "ttl": 600, + "resource_records": [{"content": [0, 0, 0, "."]}], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/_imap._tcp.unit.tests./SRV", + data={ + "ttl": 600, + "resource_records": [{"content": [0, 0, 0, "."]}], + }, + ), call( "POST", "http://api/zones/unit.tests/unit.tests./A", @@ -249,7 +358,7 @@ class TestGCoreProvider(TestCase): ] ) # expected number of total calls - self.assertEquals(7, provider._client._request.call_count) + self.assertEqual(16, provider._client._request.call_count) provider._client._request.reset_mock() @@ -283,8 +392,8 @@ class TestGCoreProvider(TestCase): plan = provider.plan(wanted) self.assertTrue(plan.exists) - self.assertEquals(2, len(plan.changes)) - self.assertEquals(2, provider.apply(plan)) + self.assertEqual(2, len(plan.changes)) + self.assertEqual(2, provider.apply(plan)) provider._client._request.assert_has_calls( [ From 006a61e4d870f1393ed21ff56964050d89646e2e Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Mon, 2 Aug 2021 18:47:11 +0300 Subject: [PATCH 13/14] add support for dynamic A, AAAA, CNAME records --- README.md | 2 +- octodns/provider/gcore.py | 248 +++++++++++++++++++-- tests/fixtures/gcore-records.json | 224 +++++++++++++++++++ tests/test_octodns_provider_gcore.py | 320 +++++++++++++++++++++++++-- 4 files changed, 764 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 9764caa..9b4031e 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | -| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA, NS, MX, TXT, SRV, CNAME, PTR | No | | +| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA, NS, MX, TXT, SRV, CNAME, PTR | Dynamic | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [HetznerProvider](/octodns/provider/hetzner.py) | | A, AAAA, CAA, CNAME, MX, NS, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index c551f03..821d109 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -15,6 +15,7 @@ import http import logging import urllib.parse +from ..record import GeoCodes from ..record import Record from .base import BaseProvider @@ -157,10 +158,11 @@ class GCoreProvider(BaseProvider): password: XXXXXXXXXXXX # auth_url: https://api.gcdn.co # url: https://dnsapi.gcorelabs.com/v2 + # records_per_response: 1 """ SUPPORTS_GEO = False - SUPPORTS_DYNAMIC = False + SUPPORTS_DYNAMIC = True SUPPORTS = set(("A", "AAAA", "NS", "MX", "TXT", "SRV", "CNAME", "PTR")) def __init__(self, id, *args, **kwargs): @@ -170,6 +172,7 @@ class GCoreProvider(BaseProvider): password = kwargs.pop("password", None) api_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") auth_url = kwargs.pop("auth_url", "https://api.gcdn.co") + self.records_per_response = kwargs.pop("records_per_response", 1) self.log = logging.getLogger("GCoreProvider[{}]".format(id)) self.log.debug("__init__: id=%s", id) super(GCoreProvider, self).__init__(id, *args, **kwargs) @@ -186,6 +189,86 @@ class GCoreProvider(BaseProvider): def _add_dot_if_need(self, value): return "{}.".format(value) if not value.endswith(".") else value + def _build_pools(self, record, default_pool_name, value_transform_fn): + defaults = [] + geo_sets, pool_idx = dict(), 0 + pools = defaultdict(lambda: {"values": []}) + for rr in record["resource_records"]: + meta = rr.get("meta", {}) + value = {"value": value_transform_fn(rr["content"][0])} + countries = meta.get("countries", []) + continents = meta.get("continents", []) + + if meta.get("default", False): + pools[default_pool_name]["values"].append(value) + defaults.append(value["value"]) + continue + # defaults is false or missing and no conties or continents + elif len(continents) == 0 and len(countries) == 0: + defaults.append(value["value"]) + continue + + # RR with the same set of countries and continents are + # combined in single pool + geo_set = frozenset( + [GeoCodes.country_to_code(cc.upper()) for cc in countries] + ) | frozenset(cc.upper() for cc in continents) + if geo_set not in geo_sets: + geo_sets[geo_set] = "pool-{}".format(pool_idx) + pool_idx += 1 + + pools[geo_sets[geo_set]]["values"].append(value) + + return pools, geo_sets, defaults + + def _build_rules(self, pools, geo_sets): + rules = [] + for name, _ in pools.items(): + rule = {"pool": name} + geo_set = next( + ( + geo_set + for geo_set, pool_name in geo_sets.items() + if pool_name == name + ), + {}, + ) + if len(geo_set) > 0: + rule["geos"] = list(geo_set) + rules.append(rule) + + return sorted(rules, key=lambda x: x["pool"]) + + def _data_for_dynamic(self, record, value_transform_fn=lambda x: x): + default_pool = "other" + pools, geo_sets, defaults = self._build_pools( + record, default_pool, value_transform_fn + ) + if len(pools) == 0: + raise RuntimeError( + "filter is enabled, but no pools where built for {}".format( + record + ) + ) + + # defaults can't be empty, so use first pool values + if len(defaults) == 0: + defaults = [ + value_transform_fn(v["value"]) + for v in next(iter(pools.values()))["values"] + ] + + # if at least one default RR was found then setup fallback for + # other pools to default + if default_pool in pools: + for pool_name, pool in pools.items(): + if pool_name == default_pool: + continue + pool["fallback"] = default_pool + + rules = self._build_rules(pools, geo_sets) + return pools, rules, defaults + def _data_for_single(self, _type, record): return { "ttl": record["ttl"], @@ -195,18 +278,42 @@ class GCoreProvider(BaseProvider): ), } - _data_for_CNAME = _data_for_single _data_for_PTR = _data_for_single - def _data_for_multiple(self, _type, record): + def _data_for_CNAME(self, _type, record): + if record.get("filters") is None: + return self._data_for_single(_type, record) + + pools, rules, defaults = self._data_for_dynamic( + record, self._add_dot_if_need + ) return { "ttl": record["ttl"], "type": _type, - "values": [ - rr_value - for resource_record in record["resource_records"] - for rr_value in resource_record["content"] - ], + "dynamic": {"pools": pools, "rules": rules}, + "value": self._add_dot_if_need(defaults[0]), + } + + def _data_for_multiple(self, _type, record): + extra = dict() + if record.get("filters") is not None: + pools, rules, defaults = self._data_for_dynamic(record) + extra = { + "dynamic": {"pools": pools, "rules": rules}, + "values": defaults, + } + else: + extra = { + "values": [ + rr_value + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ] + } + return { + "ttl": record["ttl"], + "type": _type, + **extra, } _data_for_A = _data_for_multiple @@ -286,6 +393,8 @@ class GCoreProvider(BaseProvider): _type = record["type"].upper() if _type not in self.SUPPORTS: continue + if self._should_ignore(record): + continue rr_name = zone.hostname_from_fqdn(record["name"]) values[rr_name][_type] = record @@ -309,16 +418,132 @@ class GCoreProvider(BaseProvider): ) return exists + def _should_ignore(self, record): + name = record.get("name", "name-not-defined") + if record.get("filters") is None: + return False + want_filters = 3 + filters = record.get("filters", []) + if len(filters) != want_filters: + self.log.info( + "ignore %s has filters and their count is not %d", + name, + want_filters, + ) + return True + types = [v.get("type") for v in filters] + for i, want_type in enumerate(["geodns", "first_n", "default"]): + if types[i] != want_type: + self.log.info( + "ignore %s, filters.%d.type is %s, want %s", + name, + i, + types[i], + want_type, + ) + return True + limits = [filters[i].get("limit", 1) for i in [1, 2]] + if limits[0] != limits[1]: + self.log.info( + "ignore %s, filters.1.limit (%d) != filters.2.limit (%d)", + name, + limits[0], + limits[1], + ) + return True + return False + + def _params_for_dymanic(self, record): + records = [] + default_pool_found = False + default_values = set( + record.values if hasattr(record, "values") else [record.value] + ) + for rule in record.dynamic.rules: + meta = dict() + # build meta tags if geos information present + if len(rule.data.get("geos", [])) > 0: + for geo_code in rule.data["geos"]: + geo = GeoCodes.parse(geo_code) + + country = geo["country_code"] + continent = geo["continent_code"] + if country is not None: + meta.setdefault("countries", []).append(country) + else: + meta.setdefault("continents", []).append(continent) + else: + meta["default"] = True + + pool_values = set() + pool_name = rule.data["pool"] + for value in record.dynamic.pools[pool_name].data["values"]: + v = value["value"] + records.append({"content": [v], "meta": meta}) + pool_values.add(v) + + default_pool_found |= default_values == pool_values + + # if default values doesn't match any pool values, then just add this + # values with no any meta + if not default_pool_found: + for value in default_values: + records.append({"content": [value]}) + + return records + def _params_for_single(self, record): return { "ttl": record.ttl, "resource_records": [{"content": [record.value]}], } - _params_for_CNAME = _params_for_single _params_for_PTR = _params_for_single + def _params_for_CNAME(self, record): + if not record.dynamic: + return self._params_for_single(record) + + return { + "ttl": record.ttl, + "resource_records": self._params_for_dymanic(record), + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": self.records_per_response}, + { + "type": "default", + "limit": self.records_per_response, + "strict": False, + }, + ], + } + def _params_for_multiple(self, record): + extra = dict() + if record.dynamic: + extra["resource_records"] = self._params_for_dymanic(record) + extra["filters"] = [ + {"type": "geodns"}, + {"type": "first_n", "limit": self.records_per_response}, + { + "type": "default", + "limit": self.records_per_response, + "strict": False, + }, + ] + else: + extra["resource_records"] = [ + {"content": [value]} for value in record.values + ] + return { + "ttl": record.ttl, + **extra, + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + + def _params_for_NS(self, record): return { "ttl": record.ttl, "resource_records": [ @@ -326,12 +551,7 @@ class GCoreProvider(BaseProvider): ], } - _params_for_A = _params_for_multiple - _params_for_AAAA = _params_for_multiple - _params_for_NS = _params_for_multiple - def _params_for_TXT(self, record): - # print(record.values) return { "ttl": record.ttl, "resource_records": [ diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json index ba29246..570b358 100644 --- a/tests/fixtures/gcore-records.json +++ b/tests/fixtures/gcore-records.json @@ -199,6 +199,230 @@ ] } ] + }, + { + "name": "geo-A-single.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "8.8.8.8" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "9.9.9.9" + ], + "meta": { + "continents": [ + "EU" + ] + } + }, + { + "content": [ + "10.10.10.10" + ], + "meta": { + "default": true + } + } + ] + }, + { + "name": "geo-no-def.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ], + "meta": { + "countries": [ + "RU" + ] + } + } + ] + }, + { + "name": "geo-CNAME.unit.tests.", + "type": "CNAME", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "ru-1.unit.tests" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "ru-2.unit.tests" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "eu.unit.tests" + ], + "meta": { + "continents": [ + "EU" + ] + } + }, + { + "content": [ + "any.unit.tests." + ], + "meta": { + "default": true + } + } + ] + }, + { + "name": "geo-ignore-len-filters.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] + }, + { + "name": "geo-ignore-types.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodistance" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] + }, + { + "name": "geo-ignore-limits.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 2, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] } ] } \ No newline at end of file diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 2ddbb59..14c0137 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -15,7 +15,7 @@ from requests_mock import ANY, mock as requests_mock from six import text_type from unittest import TestCase -from octodns.record import Record, Update, Delete +from octodns.record import Record, Update, Delete, Create from octodns.provider.gcore import ( GCoreProvider, GCoreClientBadRequest, @@ -35,7 +35,7 @@ class TestGCoreProvider(TestCase): provider = GCoreProvider("test_id", token="token") - # 400 - Bad Request. + # TC: 400 - Bad Request. with requests_mock() as mock: mock.get(ANY, status_code=400, text='{"error":"bad body"}') @@ -44,7 +44,7 @@ class TestGCoreProvider(TestCase): provider.populate(zone) self.assertIn('"error":"bad body"', text_type(ctx.exception)) - # 404 - Not Found. + # TC: 404 - Not Found. with requests_mock() as mock: mock.get( ANY, status_code=404, text='{"error":"zone is not found"}' @@ -57,7 +57,7 @@ class TestGCoreProvider(TestCase): '"error":"zone is not found"', text_type(ctx.exception) ) - # General error + # TC: General error with requests_mock() as mock: mock.get(ANY, status_code=500, text="Things caught fire") @@ -66,7 +66,7 @@ class TestGCoreProvider(TestCase): provider.populate(zone) self.assertEqual("Things caught fire", text_type(ctx.exception)) - # No credentials or token error + # TC: No credentials or token error with requests_mock() as mock: with self.assertRaises(ValueError) as ctx: GCoreProvider("test_id") @@ -75,7 +75,7 @@ class TestGCoreProvider(TestCase): text_type(ctx.exception), ) - # Auth with login password + # TC: Auth with login password with requests_mock() as mock: def match_body(request): @@ -108,7 +108,7 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) assert not providerPassword.populate(zone) - # No diffs == no changes + # TC: No diffs == no changes with requests_mock() as mock: base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" with open("tests/fixtures/gcore-no-changes.json") as fh: @@ -138,7 +138,7 @@ class TestGCoreProvider(TestCase): changes = self.expected.changes(zone, provider) self.assertEqual(0, len(changes)) - # 1 removed + 7 modified + # TC: 4 create (dynamic) + 1 removed + 7 modified with requests_mock() as mock: base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" with open("tests/fixtures/gcore-records.json") as fh: @@ -146,9 +146,12 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) provider.populate(zone) - self.assertEqual(13, len(zone.records)) + self.assertEqual(16, len(zone.records)) changes = self.expected.changes(zone, provider) - self.assertEqual(8, len(changes)) + self.assertEqual(11, len(changes)) + self.assertEqual( + 3, len([c for c in changes if isinstance(c, Create)]) + ) self.assertEqual( 1, len([c for c in changes if isinstance(c, Delete)]) ) @@ -156,10 +159,47 @@ class TestGCoreProvider(TestCase): 7, len([c for c in changes if isinstance(c, Update)]) ) + # TC: no pools can be built + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + mock.get( + base, + json={ + "rrsets": [ + { + "name": "unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"limit": 1, "type": "first_n"}, + { + "limit": 1, + "strict": False, + "type": "default", + }, + ], + "resource_records": [{"content": ["7.7.7.7"]}], + } + ] + }, + ) + + zone = Zone("unit.tests.", []) + with self.assertRaises(RuntimeError) as ctx: + provider.populate(zone) + + self.assertTrue( + str(ctx.exception).startswith( + "filter is enabled, but no pools where built for" + ), + "{} - is not start from desired text".format(ctx.exception), + ) + def test_apply(self): provider = GCoreProvider("test_id", url="http://api", token="token") - # Zone does not exists but can be created. + # TC: Zone does not exists but can be created. with requests_mock() as mock: mock.get( ANY, status_code=404, text='{"error":"zone is not found"}' @@ -169,7 +209,7 @@ class TestGCoreProvider(TestCase): plan = provider.plan(self.expected) provider.apply(plan) - # Zone does not exists and can't be created. + # TC: Zone does not exists and can't be created. with requests_mock() as mock: mock.get( ANY, status_code=404, text='{"error":"zone is not found"}' @@ -206,7 +246,7 @@ class TestGCoreProvider(TestCase): ] plan = provider.plan(self.expected) - # create all + # TC: create all self.assertEqual(13, len(plan.changes)) self.assertEqual(13, provider.apply(plan)) self.assertFalse(plan.exists) @@ -360,9 +400,8 @@ class TestGCoreProvider(TestCase): # expected number of total calls self.assertEqual(16, provider._client._request.call_count) + # TC: delete 1 and update 1 provider._client._request.reset_mock() - - # delete 1 and update 1 provider._client.zone_records = Mock( return_value=[ { @@ -410,3 +449,254 @@ class TestGCoreProvider(TestCase): ), ] ) + + # TC: create dynamics + provider._client._request.reset_mock() + provider._client.zone_records = Mock(return_value=[]) + + # Domain exists, we don't care about return + resp.json.side_effect = ["{}"] + + wanted = Zone("unit.tests.", []) + wanted.add_record( + Record.new( + wanted, + "geo-simple", + { + "ttl": 300, + "type": "A", + "value": "3.3.3.3", + "dynamic": { + "pools": { + "pool-1": { + "fallback": "other", + "values": [ + {"value": "1.1.1.1"}, + {"value": "1.1.1.2"}, + ], + }, + "pool-2": { + "fallback": "other", + "values": [ + {"value": "2.2.2.1"}, + ], + }, + "other": {"values": [{"value": "3.3.3.3"}]}, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU-RU"]}, + {"pool": "pool-2", "geos": ["EU"]}, + {"pool": "other"}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "geo-defaults", + { + "ttl": 300, + "type": "A", + "value": "3.2.3.4", + "dynamic": { + "pools": { + "pool-1": { + "values": [ + {"value": "2.2.2.1"}, + ], + }, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU"]}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "geo-cname-simple", + { + "ttl": 300, + "type": "CNAME", + "value": "en.unit.tests.", + "dynamic": { + "pools": { + "pool-1": { + "fallback": "other", + "values": [ + {"value": "ru-1.unit.tests."}, + {"value": "ru-2.unit.tests."}, + ], + }, + "pool-2": { + "fallback": "other", + "values": [ + {"value": "eu.unit.tests."}, + ], + }, + "other": {"values": [{"value": "en.unit.tests."}]}, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU-RU"]}, + {"pool": "pool-2", "geos": ["EU"]}, + {"pool": "other"}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "geo-cname-defaults", + { + "ttl": 300, + "type": "CNAME", + "value": "en.unit.tests.", + "dynamic": { + "pools": { + "pool-1": { + "values": [ + {"value": "eu.unit.tests."}, + ], + }, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU"]}, + ], + }, + }, + ), + ) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEqual(4, len(plan.changes)) + self.assertEqual(4, provider.apply(plan)) + + provider._client._request.assert_has_calls( + [ + call( + "POST", + "http://api/zones/unit.tests/geo-simple.unit.tests./A", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["1.1.1.1"], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["1.1.1.2"], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["2.2.2.1"], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["3.3.3.3"], + "meta": {"default": True}, + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/geo-defaults.unit.tests./A", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["2.2.2.1"], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["3.2.3.4"], + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/geo-cname-simple.unit.tests./CNAME", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["ru-1.unit.tests."], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["ru-2.unit.tests."], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["eu.unit.tests."], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["en.unit.tests."], + "meta": {"default": True}, + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/geo-cname-defaults.unit.tests./CNAME", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["eu.unit.tests."], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["en.unit.tests."], + }, + ], + }, + ), + ] + ) From 33d56b8357ebde9b513925e929440c581c92c03a Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Tue, 3 Aug 2021 10:46:04 +0300 Subject: [PATCH 14/14] filters must be ordered as 'geodns', 'defaults', 'first_n' --- octodns/provider/gcore.py | 12 ++--- tests/fixtures/gcore-records.json | 26 +++++------ tests/test_octodns_provider_gcore.py | 68 ++++++++-------------------- 3 files changed, 38 insertions(+), 68 deletions(-) diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index 821d109..bbbf81f 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -194,10 +194,10 @@ class GCoreProvider(BaseProvider): geo_sets, pool_idx = dict(), 0 pools = defaultdict(lambda: {"values": []}) for rr in record["resource_records"]: - meta = rr.get("meta", {}) + meta = rr.get("meta", {}) or {} value = {"value": value_transform_fn(rr["content"][0])} - countries = meta.get("countries", []) - continents = meta.get("continents", []) + countries = meta.get("countries", []) or [] + continents = meta.get("continents", []) or [] if meta.get("default", False): pools[default_pool_name]["values"].append(value) @@ -432,7 +432,7 @@ class GCoreProvider(BaseProvider): ) return True types = [v.get("type") for v in filters] - for i, want_type in enumerate(["geodns", "first_n", "default"]): + for i, want_type in enumerate(["geodns", "default", "first_n"]): if types[i] != want_type: self.log.info( "ignore %s, filters.%d.type is %s, want %s", @@ -509,12 +509,12 @@ class GCoreProvider(BaseProvider): "resource_records": self._params_for_dymanic(record), "filters": [ {"type": "geodns"}, - {"type": "first_n", "limit": self.records_per_response}, { "type": "default", "limit": self.records_per_response, "strict": False, }, + {"type": "first_n", "limit": self.records_per_response}, ], } @@ -524,12 +524,12 @@ class GCoreProvider(BaseProvider): extra["resource_records"] = self._params_for_dymanic(record) extra["filters"] = [ {"type": "geodns"}, - {"type": "first_n", "limit": self.records_per_response}, { "type": "default", "limit": self.records_per_response, "strict": False, }, + {"type": "first_n", "limit": self.records_per_response}, ] else: extra["resource_records"] = [ diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json index 570b358..9bf58d7 100644 --- a/tests/fixtures/gcore-records.json +++ b/tests/fixtures/gcore-records.json @@ -210,12 +210,12 @@ }, { "limit": 1, - "type": "first_n" + "strict": false, + "type": "default" }, { "limit": 1, - "strict": false, - "type": "default" + "type": "first_n" } ], "resource_records": [ @@ -269,12 +269,12 @@ }, { "limit": 1, - "type": "first_n" + "strict": false, + "type": "default" }, { "limit": 1, - "strict": false, - "type": "default" + "type": "first_n" } ], "resource_records": [ @@ -300,12 +300,12 @@ }, { "limit": 1, - "type": "first_n" + "strict": false, + "type": "default" }, { "limit": 1, - "strict": false, - "type": "default" + "type": "first_n" } ], "resource_records": [ @@ -406,14 +406,14 @@ { "type": "geodns" }, - { - "limit": 1, - "type": "first_n" - }, { "limit": 2, "strict": false, "type": "default" + }, + { + "limit": 1, + "type": "first_n" } ], "resource_records": [ diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 14c0137..2151440 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -31,6 +31,16 @@ class TestGCoreProvider(TestCase): source = YamlProvider("test", join(dirname(__file__), "config")) source.populate(expected) + default_filters = [ + {"type": "geodns"}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + {"type": "first_n", "limit": 1}, + ] + def test_populate(self): provider = GCoreProvider("test_id", token="token") @@ -170,15 +180,7 @@ class TestGCoreProvider(TestCase): "name": "unit.tests.", "type": "A", "ttl": 300, - "filters": [ - {"type": "geodns"}, - {"limit": 1, "type": "first_n"}, - { - "limit": 1, - "strict": False, - "type": "default", - }, - ], + "filters": self.default_filters, "resource_records": [{"content": ["7.7.7.7"]}], } ] @@ -518,7 +520,7 @@ class TestGCoreProvider(TestCase): wanted.add_record( Record.new( wanted, - "geo-cname-simple", + "cname-smpl", { "ttl": 300, "type": "CNAME", @@ -552,7 +554,7 @@ class TestGCoreProvider(TestCase): wanted.add_record( Record.new( wanted, - "geo-cname-defaults", + "cname-dflt", { "ttl": 300, "type": "CNAME", @@ -585,15 +587,7 @@ class TestGCoreProvider(TestCase): "http://api/zones/unit.tests/geo-simple.unit.tests./A", data={ "ttl": 300, - "filters": [ - {"type": "geodns"}, - {"type": "first_n", "limit": 1}, - { - "type": "default", - "limit": 1, - "strict": False, - }, - ], + "filters": self.default_filters, "resource_records": [ { "content": ["1.1.1.1"], @@ -619,15 +613,7 @@ class TestGCoreProvider(TestCase): "http://api/zones/unit.tests/geo-defaults.unit.tests./A", data={ "ttl": 300, - "filters": [ - {"type": "geodns"}, - {"type": "first_n", "limit": 1}, - { - "type": "default", - "limit": 1, - "strict": False, - }, - ], + "filters": self.default_filters, "resource_records": [ { "content": ["2.2.2.1"], @@ -641,18 +627,10 @@ class TestGCoreProvider(TestCase): ), call( "POST", - "http://api/zones/unit.tests/geo-cname-simple.unit.tests./CNAME", + "http://api/zones/unit.tests/cname-smpl.unit.tests./CNAME", data={ "ttl": 300, - "filters": [ - {"type": "geodns"}, - {"type": "first_n", "limit": 1}, - { - "type": "default", - "limit": 1, - "strict": False, - }, - ], + "filters": self.default_filters, "resource_records": [ { "content": ["ru-1.unit.tests."], @@ -675,18 +653,10 @@ class TestGCoreProvider(TestCase): ), call( "POST", - "http://api/zones/unit.tests/geo-cname-defaults.unit.tests./CNAME", + "http://api/zones/unit.tests/cname-dflt.unit.tests./CNAME", data={ "ttl": 300, - "filters": [ - {"type": "geodns"}, - {"type": "first_n", "limit": 1}, - { - "type": "default", - "limit": 1, - "strict": False, - }, - ], + "filters": self.default_filters, "resource_records": [ { "content": ["eu.unit.tests."],