From 006a61e4d870f1393ed21ff56964050d89646e2e Mon Sep 17 00:00:00 2001 From: "Yaroshevich, Denis" Date: Mon, 2 Aug 2021 18:47:11 +0300 Subject: [PATCH] add support for dynamic A, AAAA, CNAME records --- README.md | 2 +- octodns/provider/gcore.py | 248 +++++++++++++++++++-- tests/fixtures/gcore-records.json | 224 +++++++++++++++++++ tests/test_octodns_provider_gcore.py | 320 +++++++++++++++++++++++++-- 4 files changed, 764 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 9764caa..9b4031e 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | -| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA, NS, MX, TXT, SRV, CNAME, PTR | No | | +| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA, NS, MX, TXT, SRV, CNAME, PTR | Dynamic | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [HetznerProvider](/octodns/provider/hetzner.py) | | A, AAAA, CAA, CNAME, MX, NS, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py index c551f03..821d109 100644 --- a/octodns/provider/gcore.py +++ b/octodns/provider/gcore.py @@ -15,6 +15,7 @@ import http import logging import urllib.parse +from ..record import GeoCodes from ..record import Record from .base import BaseProvider @@ -157,10 +158,11 @@ class GCoreProvider(BaseProvider): password: XXXXXXXXXXXX # auth_url: https://api.gcdn.co # url: https://dnsapi.gcorelabs.com/v2 + # records_per_response: 1 """ SUPPORTS_GEO = False - SUPPORTS_DYNAMIC = False + SUPPORTS_DYNAMIC = True SUPPORTS = set(("A", "AAAA", "NS", "MX", "TXT", "SRV", "CNAME", "PTR")) def __init__(self, id, *args, **kwargs): @@ -170,6 +172,7 @@ class GCoreProvider(BaseProvider): password = kwargs.pop("password", None) api_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") auth_url = kwargs.pop("auth_url", "https://api.gcdn.co") + self.records_per_response = kwargs.pop("records_per_response", 1) self.log = logging.getLogger("GCoreProvider[{}]".format(id)) self.log.debug("__init__: id=%s", id) super(GCoreProvider, self).__init__(id, *args, **kwargs) @@ -186,6 +189,86 @@ class GCoreProvider(BaseProvider): def _add_dot_if_need(self, value): return "{}.".format(value) if not value.endswith(".") else value + def _build_pools(self, record, default_pool_name, value_transform_fn): + defaults = [] + geo_sets, pool_idx = dict(), 0 + pools = defaultdict(lambda: {"values": []}) + for rr in record["resource_records"]: + meta = rr.get("meta", {}) + value = {"value": value_transform_fn(rr["content"][0])} + countries = meta.get("countries", []) + continents = meta.get("continents", []) + + if meta.get("default", False): + pools[default_pool_name]["values"].append(value) + defaults.append(value["value"]) + continue + # defaults is false or missing and no conties or continents + elif len(continents) == 0 and len(countries) == 0: + defaults.append(value["value"]) + continue + + # RR with the same set of countries and continents are + # combined in single pool + geo_set = frozenset( + [GeoCodes.country_to_code(cc.upper()) for cc in countries] + ) | frozenset(cc.upper() for cc in continents) + if geo_set not in geo_sets: + geo_sets[geo_set] = "pool-{}".format(pool_idx) + pool_idx += 1 + + pools[geo_sets[geo_set]]["values"].append(value) + + return pools, geo_sets, defaults + + def _build_rules(self, pools, geo_sets): + rules = [] + for name, _ in pools.items(): + rule = {"pool": name} + geo_set = next( + ( + geo_set + for geo_set, pool_name in geo_sets.items() + if pool_name == name + ), + {}, + ) + if len(geo_set) > 0: + rule["geos"] = list(geo_set) + rules.append(rule) + + return sorted(rules, key=lambda x: x["pool"]) + + def _data_for_dynamic(self, record, value_transform_fn=lambda x: x): + default_pool = "other" + pools, geo_sets, defaults = self._build_pools( + record, default_pool, value_transform_fn + ) + if len(pools) == 0: + raise RuntimeError( + "filter is enabled, but no pools where built for {}".format( + record + ) + ) + + # defaults can't be empty, so use first pool values + if len(defaults) == 0: + defaults = [ + value_transform_fn(v["value"]) + for v in next(iter(pools.values()))["values"] + ] + + # if at least one default RR was found then setup fallback for + # other pools to default + if default_pool in pools: + for pool_name, pool in pools.items(): + if pool_name == default_pool: + continue + pool["fallback"] = default_pool + + rules = self._build_rules(pools, geo_sets) + return pools, rules, defaults + def _data_for_single(self, _type, record): return { "ttl": record["ttl"], @@ -195,18 +278,42 @@ class GCoreProvider(BaseProvider): ), } - _data_for_CNAME = _data_for_single _data_for_PTR = _data_for_single - def _data_for_multiple(self, _type, record): + def _data_for_CNAME(self, _type, record): + if record.get("filters") is None: + return self._data_for_single(_type, record) + + pools, rules, defaults = self._data_for_dynamic( + record, self._add_dot_if_need + ) return { "ttl": record["ttl"], "type": _type, - "values": [ - rr_value - for resource_record in record["resource_records"] - for rr_value in resource_record["content"] - ], + "dynamic": {"pools": pools, "rules": rules}, + "value": self._add_dot_if_need(defaults[0]), + } + + def _data_for_multiple(self, _type, record): + extra = dict() + if record.get("filters") is not None: + pools, rules, defaults = self._data_for_dynamic(record) + extra = { + "dynamic": {"pools": pools, "rules": rules}, + "values": defaults, + } + else: + extra = { + "values": [ + rr_value + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ] + } + return { + "ttl": record["ttl"], + "type": _type, + **extra, } _data_for_A = _data_for_multiple @@ -286,6 +393,8 @@ class GCoreProvider(BaseProvider): _type = record["type"].upper() if _type not in self.SUPPORTS: continue + if self._should_ignore(record): + continue rr_name = zone.hostname_from_fqdn(record["name"]) values[rr_name][_type] = record @@ -309,16 +418,132 @@ class GCoreProvider(BaseProvider): ) return exists + def _should_ignore(self, record): + name = record.get("name", "name-not-defined") + if record.get("filters") is None: + return False + want_filters = 3 + filters = record.get("filters", []) + if len(filters) != want_filters: + self.log.info( + "ignore %s has filters and their count is not %d", + name, + want_filters, + ) + return True + types = [v.get("type") for v in filters] + for i, want_type in enumerate(["geodns", "first_n", "default"]): + if types[i] != want_type: + self.log.info( + "ignore %s, filters.%d.type is %s, want %s", + name, + i, + types[i], + want_type, + ) + return True + limits = [filters[i].get("limit", 1) for i in [1, 2]] + if limits[0] != limits[1]: + self.log.info( + "ignore %s, filters.1.limit (%d) != filters.2.limit (%d)", + name, + limits[0], + limits[1], + ) + return True + return False + + def _params_for_dymanic(self, record): + records = [] + default_pool_found = False + default_values = set( + record.values if hasattr(record, "values") else [record.value] + ) + for rule in record.dynamic.rules: + meta = dict() + # build meta tags if geos information present + if len(rule.data.get("geos", [])) > 0: + for geo_code in rule.data["geos"]: + geo = GeoCodes.parse(geo_code) + + country = geo["country_code"] + continent = geo["continent_code"] + if country is not None: + meta.setdefault("countries", []).append(country) + else: + meta.setdefault("continents", []).append(continent) + else: + meta["default"] = True + + pool_values = set() + pool_name = rule.data["pool"] + for value in record.dynamic.pools[pool_name].data["values"]: + v = value["value"] + records.append({"content": [v], "meta": meta}) + pool_values.add(v) + + default_pool_found |= default_values == pool_values + + # if default values doesn't match any pool values, then just add this + # values with no any meta + if not default_pool_found: + for value in default_values: + records.append({"content": [value]}) + + return records + def _params_for_single(self, record): return { "ttl": record.ttl, "resource_records": [{"content": [record.value]}], } - _params_for_CNAME = _params_for_single _params_for_PTR = _params_for_single + def _params_for_CNAME(self, record): + if not record.dynamic: + return self._params_for_single(record) + + return { + "ttl": record.ttl, + "resource_records": self._params_for_dymanic(record), + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": self.records_per_response}, + { + "type": "default", + "limit": self.records_per_response, + "strict": False, + }, + ], + } + def _params_for_multiple(self, record): + extra = dict() + if record.dynamic: + extra["resource_records"] = self._params_for_dymanic(record) + extra["filters"] = [ + {"type": "geodns"}, + {"type": "first_n", "limit": self.records_per_response}, + { + "type": "default", + "limit": self.records_per_response, + "strict": False, + }, + ] + else: + extra["resource_records"] = [ + {"content": [value]} for value in record.values + ] + return { + "ttl": record.ttl, + **extra, + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + + def _params_for_NS(self, record): return { "ttl": record.ttl, "resource_records": [ @@ -326,12 +551,7 @@ class GCoreProvider(BaseProvider): ], } - _params_for_A = _params_for_multiple - _params_for_AAAA = _params_for_multiple - _params_for_NS = _params_for_multiple - def _params_for_TXT(self, record): - # print(record.values) return { "ttl": record.ttl, "resource_records": [ diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json index ba29246..570b358 100644 --- a/tests/fixtures/gcore-records.json +++ b/tests/fixtures/gcore-records.json @@ -199,6 +199,230 @@ ] } ] + }, + { + "name": "geo-A-single.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "8.8.8.8" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "9.9.9.9" + ], + "meta": { + "continents": [ + "EU" + ] + } + }, + { + "content": [ + "10.10.10.10" + ], + "meta": { + "default": true + } + } + ] + }, + { + "name": "geo-no-def.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ], + "meta": { + "countries": [ + "RU" + ] + } + } + ] + }, + { + "name": "geo-CNAME.unit.tests.", + "type": "CNAME", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "ru-1.unit.tests" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "ru-2.unit.tests" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "eu.unit.tests" + ], + "meta": { + "continents": [ + "EU" + ] + } + }, + { + "content": [ + "any.unit.tests." + ], + "meta": { + "default": true + } + } + ] + }, + { + "name": "geo-ignore-len-filters.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] + }, + { + "name": "geo-ignore-types.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodistance" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] + }, + { + "name": "geo-ignore-limits.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 2, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] } ] } \ No newline at end of file diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py index 2ddbb59..14c0137 100644 --- a/tests/test_octodns_provider_gcore.py +++ b/tests/test_octodns_provider_gcore.py @@ -15,7 +15,7 @@ from requests_mock import ANY, mock as requests_mock from six import text_type from unittest import TestCase -from octodns.record import Record, Update, Delete +from octodns.record import Record, Update, Delete, Create from octodns.provider.gcore import ( GCoreProvider, GCoreClientBadRequest, @@ -35,7 +35,7 @@ class TestGCoreProvider(TestCase): provider = GCoreProvider("test_id", token="token") - # 400 - Bad Request. + # TC: 400 - Bad Request. with requests_mock() as mock: mock.get(ANY, status_code=400, text='{"error":"bad body"}') @@ -44,7 +44,7 @@ class TestGCoreProvider(TestCase): provider.populate(zone) self.assertIn('"error":"bad body"', text_type(ctx.exception)) - # 404 - Not Found. + # TC: 404 - Not Found. with requests_mock() as mock: mock.get( ANY, status_code=404, text='{"error":"zone is not found"}' @@ -57,7 +57,7 @@ class TestGCoreProvider(TestCase): '"error":"zone is not found"', text_type(ctx.exception) ) - # General error + # TC: General error with requests_mock() as mock: mock.get(ANY, status_code=500, text="Things caught fire") @@ -66,7 +66,7 @@ class TestGCoreProvider(TestCase): provider.populate(zone) self.assertEqual("Things caught fire", text_type(ctx.exception)) - # No credentials or token error + # TC: No credentials or token error with requests_mock() as mock: with self.assertRaises(ValueError) as ctx: GCoreProvider("test_id") @@ -75,7 +75,7 @@ class TestGCoreProvider(TestCase): text_type(ctx.exception), ) - # Auth with login password + # TC: Auth with login password with requests_mock() as mock: def match_body(request): @@ -108,7 +108,7 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) assert not providerPassword.populate(zone) - # No diffs == no changes + # TC: No diffs == no changes with requests_mock() as mock: base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" with open("tests/fixtures/gcore-no-changes.json") as fh: @@ -138,7 +138,7 @@ class TestGCoreProvider(TestCase): changes = self.expected.changes(zone, provider) self.assertEqual(0, len(changes)) - # 1 removed + 7 modified + # TC: 4 create (dynamic) + 1 removed + 7 modified with requests_mock() as mock: base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" with open("tests/fixtures/gcore-records.json") as fh: @@ -146,9 +146,12 @@ class TestGCoreProvider(TestCase): zone = Zone("unit.tests.", []) provider.populate(zone) - self.assertEqual(13, len(zone.records)) + self.assertEqual(16, len(zone.records)) changes = self.expected.changes(zone, provider) - self.assertEqual(8, len(changes)) + self.assertEqual(11, len(changes)) + self.assertEqual( + 3, len([c for c in changes if isinstance(c, Create)]) + ) self.assertEqual( 1, len([c for c in changes if isinstance(c, Delete)]) ) @@ -156,10 +159,47 @@ class TestGCoreProvider(TestCase): 7, len([c for c in changes if isinstance(c, Update)]) ) + # TC: no pools can be built + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + mock.get( + base, + json={ + "rrsets": [ + { + "name": "unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"limit": 1, "type": "first_n"}, + { + "limit": 1, + "strict": False, + "type": "default", + }, + ], + "resource_records": [{"content": ["7.7.7.7"]}], + } + ] + }, + ) + + zone = Zone("unit.tests.", []) + with self.assertRaises(RuntimeError) as ctx: + provider.populate(zone) + + self.assertTrue( + str(ctx.exception).startswith( + "filter is enabled, but no pools where built for" + ), + "{} - is not start from desired text".format(ctx.exception), + ) + def test_apply(self): provider = GCoreProvider("test_id", url="http://api", token="token") - # Zone does not exists but can be created. + # TC: Zone does not exists but can be created. with requests_mock() as mock: mock.get( ANY, status_code=404, text='{"error":"zone is not found"}' @@ -169,7 +209,7 @@ class TestGCoreProvider(TestCase): plan = provider.plan(self.expected) provider.apply(plan) - # Zone does not exists and can't be created. + # TC: Zone does not exists and can't be created. with requests_mock() as mock: mock.get( ANY, status_code=404, text='{"error":"zone is not found"}' @@ -206,7 +246,7 @@ class TestGCoreProvider(TestCase): ] plan = provider.plan(self.expected) - # create all + # TC: create all self.assertEqual(13, len(plan.changes)) self.assertEqual(13, provider.apply(plan)) self.assertFalse(plan.exists) @@ -360,9 +400,8 @@ class TestGCoreProvider(TestCase): # expected number of total calls self.assertEqual(16, provider._client._request.call_count) + # TC: delete 1 and update 1 provider._client._request.reset_mock() - - # delete 1 and update 1 provider._client.zone_records = Mock( return_value=[ { @@ -410,3 +449,254 @@ class TestGCoreProvider(TestCase): ), ] ) + + # TC: create dynamics + provider._client._request.reset_mock() + provider._client.zone_records = Mock(return_value=[]) + + # Domain exists, we don't care about return + resp.json.side_effect = ["{}"] + + wanted = Zone("unit.tests.", []) + wanted.add_record( + Record.new( + wanted, + "geo-simple", + { + "ttl": 300, + "type": "A", + "value": "3.3.3.3", + "dynamic": { + "pools": { + "pool-1": { + "fallback": "other", + "values": [ + {"value": "1.1.1.1"}, + {"value": "1.1.1.2"}, + ], + }, + "pool-2": { + "fallback": "other", + "values": [ + {"value": "2.2.2.1"}, + ], + }, + "other": {"values": [{"value": "3.3.3.3"}]}, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU-RU"]}, + {"pool": "pool-2", "geos": ["EU"]}, + {"pool": "other"}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "geo-defaults", + { + "ttl": 300, + "type": "A", + "value": "3.2.3.4", + "dynamic": { + "pools": { + "pool-1": { + "values": [ + {"value": "2.2.2.1"}, + ], + }, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU"]}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "geo-cname-simple", + { + "ttl": 300, + "type": "CNAME", + "value": "en.unit.tests.", + "dynamic": { + "pools": { + "pool-1": { + "fallback": "other", + "values": [ + {"value": "ru-1.unit.tests."}, + {"value": "ru-2.unit.tests."}, + ], + }, + "pool-2": { + "fallback": "other", + "values": [ + {"value": "eu.unit.tests."}, + ], + }, + "other": {"values": [{"value": "en.unit.tests."}]}, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU-RU"]}, + {"pool": "pool-2", "geos": ["EU"]}, + {"pool": "other"}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "geo-cname-defaults", + { + "ttl": 300, + "type": "CNAME", + "value": "en.unit.tests.", + "dynamic": { + "pools": { + "pool-1": { + "values": [ + {"value": "eu.unit.tests."}, + ], + }, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU"]}, + ], + }, + }, + ), + ) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEqual(4, len(plan.changes)) + self.assertEqual(4, provider.apply(plan)) + + provider._client._request.assert_has_calls( + [ + call( + "POST", + "http://api/zones/unit.tests/geo-simple.unit.tests./A", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["1.1.1.1"], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["1.1.1.2"], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["2.2.2.1"], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["3.3.3.3"], + "meta": {"default": True}, + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/geo-defaults.unit.tests./A", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["2.2.2.1"], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["3.2.3.4"], + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/geo-cname-simple.unit.tests./CNAME", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["ru-1.unit.tests."], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["ru-2.unit.tests."], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["eu.unit.tests."], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["en.unit.tests."], + "meta": {"default": True}, + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/geo-cname-defaults.unit.tests./CNAME", + data={ + "ttl": 300, + "filters": [ + {"type": "geodns"}, + {"type": "first_n", "limit": 1}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + ], + "resource_records": [ + { + "content": ["eu.unit.tests."], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["en.unit.tests."], + }, + ], + }, + ), + ] + )