diff --git a/CHANGELOG.md b/CHANGELOG.md index 55a88c2..ca42a68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ * NS1 NA target now includes `SX` and `UM`. If `NA` continent is in use in dynamic records care must be taken to upgrade/downgrade to v0.9.13. +* Ns1Provider now supports a new parameter, shared_notifylist, which results in + all dynamic record monitors using a shared notify list named 'octoDNS NS1 + Notify List'. Only newly created record values will use the shared notify + list. It should be safe to enable this functionality, but existing records + will not be converted. Note: Once this option is enabled downgrades to + previous versions of octoDNS are discouraged and may result in undefined + behavior and broken records. See https://github.com/octodns/octodns/pull/749 + for related discussion. ## v0.9.13 - 2021-07-18 - Processors Alpha @@ -17,7 +25,7 @@ * Fixes NS1 provider's geotarget limitation of using `NA` continent. Now, when `NA` is used in geos it considers **all** the countries of `North America` insted of just `us-east`, `us-west` and `us-central` regions -* `SX' & 'UM` country support added to NS1Provider, not yet in the North +* `SX' & 'UM` country support added to NS1Provider, not yet in the North America list for backwards compatibility reasons. They will be added in the next releaser. diff --git a/README.md b/README.md index 684e496..1f0e9e3 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,7 @@ The above command pulled the existing data out of Route53 and placed the results | [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | | | [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection | | [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | | +| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA, NS, MX, TXT, SRV, CNAME, PTR | Dynamic | | | [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | | | [HetznerProvider](/octodns/provider/hetzner.py) | | A, AAAA, CAA, CNAME, MX, NS, SRV, TXT | No | | | [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | | diff --git a/octodns/provider/gcore.py b/octodns/provider/gcore.py new file mode 100644 index 0000000..bbbf81f --- /dev/null +++ b/octodns/provider/gcore.py @@ -0,0 +1,623 @@ +# +# +# + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from collections import defaultdict +from requests import Session +import http +import logging +import urllib.parse + +from ..record import GeoCodes +from ..record import Record +from .base import BaseProvider + + +class GCoreClientException(Exception): + def __init__(self, r): + super(GCoreClientException, self).__init__(r.text) + + +class GCoreClientBadRequest(GCoreClientException): + def __init__(self, r): + super(GCoreClientBadRequest, self).__init__(r) + + +class GCoreClientNotFound(GCoreClientException): + def __init__(self, r): + super(GCoreClientNotFound, self).__init__(r) + + +class GCoreClient(object): + + ROOT_ZONES = "zones" + + def __init__( + self, + log, + api_url, + auth_url, + token=None, + token_type=None, + login=None, + password=None, + ): + self.log = log + self._session = Session() + self._api_url = api_url + if token is not None and token_type is not None: + self._session.headers.update( + {"Authorization": "{} {}".format(token_type, token)} + ) + elif login is not None and password is not None: + token = self._auth(auth_url, login, password) + self._session.headers.update( + {"Authorization": "Bearer {}".format(token)} + ) + else: + raise ValueError("either token or login & password must be set") + + def _auth(self, url, login, password): + # well, can't use _request, since API returns 400 if credentials + # invalid which will be logged, but we don't want do this + r = self._session.request( + "POST", + self._build_url(url, "auth", "jwt", "login"), + json={"username": login, "password": password}, + ) + r.raise_for_status() + return r.json()["access"] + + def _request(self, method, url, params=None, data=None): + r = self._session.request( + method, url, params=params, json=data, timeout=30.0 + ) + if r.status_code == http.HTTPStatus.BAD_REQUEST: + self.log.error( + "bad request %r has been sent to %r: %s", data, url, r.text + ) + raise GCoreClientBadRequest(r) + elif r.status_code == http.HTTPStatus.NOT_FOUND: + self.log.error("resource %r not found: %s", url, r.text) + raise GCoreClientNotFound(r) + elif r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR: + self.log.error("server error no %r to %r: %s", data, url, r.text) + raise GCoreClientException(r) + r.raise_for_status() + return r + + def zone(self, zone_name): + return self._request( + "GET", self._build_url(self._api_url, self.ROOT_ZONES, zone_name) + ).json() + + def zone_create(self, zone_name): + return self._request( + "POST", + self._build_url(self._api_url, self.ROOT_ZONES), + data={"name": zone_name}, + ).json() + + def zone_records(self, zone_name): + rrsets = self._request( + "GET", + "{}".format( + self._build_url( + self._api_url, self.ROOT_ZONES, zone_name, "rrsets" + ) + ), + params={"all": "true"}, + ).json() + records = rrsets["rrsets"] + return records + + def record_create(self, zone_name, rrset_name, type_, data): + self._request( + "POST", self._rrset_url(zone_name, rrset_name, type_), data=data + ) + + def record_update(self, zone_name, rrset_name, type_, data): + self._request( + "PUT", self._rrset_url(zone_name, rrset_name, type_), data=data + ) + + def record_delete(self, zone_name, rrset_name, type_): + self._request("DELETE", self._rrset_url(zone_name, rrset_name, type_)) + + def _rrset_url(self, zone_name, rrset_name, type_): + return self._build_url( + self._api_url, self.ROOT_ZONES, zone_name, rrset_name, type_ + ) + + @staticmethod + def _build_url(base, *items): + for i in items: + base = base.strip("/") + "/" + base = urllib.parse.urljoin(base, i) + return base + + +class GCoreProvider(BaseProvider): + """ + GCore provider using API v2. + + gcore: + class: octodns.provider.gcore.GCoreProvider + # Your API key + token: XXXXXXXXXXXX + # token_type: APIKey + # or login + password + login: XXXXXXXXXXXX + password: XXXXXXXXXXXX + # auth_url: https://api.gcdn.co + # url: https://dnsapi.gcorelabs.com/v2 + # records_per_response: 1 + """ + + SUPPORTS_GEO = False + SUPPORTS_DYNAMIC = True + SUPPORTS = set(("A", "AAAA", "NS", "MX", "TXT", "SRV", "CNAME", "PTR")) + + def __init__(self, id, *args, **kwargs): + token = kwargs.pop("token", None) + token_type = kwargs.pop("token_type", "APIKey") + login = kwargs.pop("login", None) + password = kwargs.pop("password", None) + api_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2") + auth_url = kwargs.pop("auth_url", "https://api.gcdn.co") + self.records_per_response = kwargs.pop("records_per_response", 1) + self.log = logging.getLogger("GCoreProvider[{}]".format(id)) + self.log.debug("__init__: id=%s", id) + super(GCoreProvider, self).__init__(id, *args, **kwargs) + self._client = GCoreClient( + self.log, + api_url, + auth_url, + token=token, + token_type=token_type, + login=login, + password=password, + ) + + def _add_dot_if_need(self, value): + return "{}.".format(value) if not value.endswith(".") else value + + def _build_pools(self, record, default_pool_name, value_transform_fn): + defaults = [] + geo_sets, pool_idx = dict(), 0 + pools = defaultdict(lambda: {"values": []}) + for rr in record["resource_records"]: + meta = rr.get("meta", {}) or {} + value = {"value": value_transform_fn(rr["content"][0])} + countries = meta.get("countries", []) or [] + continents = meta.get("continents", []) or [] + + if meta.get("default", False): + pools[default_pool_name]["values"].append(value) + defaults.append(value["value"]) + continue + # defaults is false or missing and no conties or continents + elif len(continents) == 0 and len(countries) == 0: + defaults.append(value["value"]) + continue + + # RR with the same set of countries and continents are + # combined in single pool + geo_set = frozenset( + [GeoCodes.country_to_code(cc.upper()) for cc in countries] + ) | frozenset(cc.upper() for cc in continents) + if geo_set not in geo_sets: + geo_sets[geo_set] = "pool-{}".format(pool_idx) + pool_idx += 1 + + pools[geo_sets[geo_set]]["values"].append(value) + + return pools, geo_sets, defaults + + def _build_rules(self, pools, geo_sets): + rules = [] + for name, _ in pools.items(): + rule = {"pool": name} + geo_set = next( + ( + geo_set + for geo_set, pool_name in geo_sets.items() + if pool_name == name + ), + {}, + ) + if len(geo_set) > 0: + rule["geos"] = list(geo_set) + rules.append(rule) + + return sorted(rules, key=lambda x: x["pool"]) + + def _data_for_dynamic(self, record, value_transform_fn=lambda x: x): + default_pool = "other" + pools, geo_sets, defaults = self._build_pools( + record, default_pool, value_transform_fn + ) + if len(pools) == 0: + raise RuntimeError( + "filter is enabled, but no pools where built for {}".format( + record + ) + ) + + # defaults can't be empty, so use first pool values + if len(defaults) == 0: + defaults = [ + value_transform_fn(v["value"]) + for v in next(iter(pools.values()))["values"] + ] + + # if at least one default RR was found then setup fallback for + # other pools to default + if default_pool in pools: + for pool_name, pool in pools.items(): + if pool_name == default_pool: + continue + pool["fallback"] = default_pool + + rules = self._build_rules(pools, geo_sets) + return pools, rules, defaults + + def _data_for_single(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "value": self._add_dot_if_need( + record["resource_records"][0]["content"][0] + ), + } + + _data_for_PTR = _data_for_single + + def _data_for_CNAME(self, _type, record): + if record.get("filters") is None: + return self._data_for_single(_type, record) + + pools, rules, defaults = self._data_for_dynamic( + record, self._add_dot_if_need + ) + return { + "ttl": record["ttl"], + "type": _type, + "dynamic": {"pools": pools, "rules": rules}, + "value": self._add_dot_if_need(defaults[0]), + } + + def _data_for_multiple(self, _type, record): + extra = dict() + if record.get("filters") is not None: + pools, rules, defaults = self._data_for_dynamic(record) + extra = { + "dynamic": {"pools": pools, "rules": rules}, + "values": defaults, + } + else: + extra = { + "values": [ + rr_value + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ] + } + return { + "ttl": record["ttl"], + "type": _type, + **extra, + } + + _data_for_A = _data_for_multiple + _data_for_AAAA = _data_for_multiple + + def _data_for_TXT(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + rr_value.replace(";", "\\;") + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ], + } + + def _data_for_MX(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + dict( + preference=preference, + exchange=self._add_dot_if_need(exchange), + ) + for preference, exchange in map( + lambda x: x["content"], record["resource_records"] + ) + ], + } + + def _data_for_NS(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + self._add_dot_if_need(rr_value) + for resource_record in record["resource_records"] + for rr_value in resource_record["content"] + ], + } + + def _data_for_SRV(self, _type, record): + return { + "ttl": record["ttl"], + "type": _type, + "values": [ + dict( + priority=priority, + weight=weight, + port=port, + target=self._add_dot_if_need(target), + ) + for priority, weight, port, target in map( + lambda x: x["content"], record["resource_records"] + ) + ], + } + + def zone_records(self, zone): + try: + return self._client.zone_records(zone.name[:-1]), True + except GCoreClientNotFound: + return [], False + + def populate(self, zone, target=False, lenient=False): + self.log.debug( + "populate: name=%s, target=%s, lenient=%s", + zone.name, + target, + lenient, + ) + + values = defaultdict(defaultdict) + records, exists = self.zone_records(zone) + for record in records: + _type = record["type"].upper() + if _type not in self.SUPPORTS: + continue + if self._should_ignore(record): + continue + rr_name = zone.hostname_from_fqdn(record["name"]) + values[rr_name][_type] = record + + before = len(zone.records) + for name, types in values.items(): + for _type, record in types.items(): + data_for = getattr(self, "_data_for_{}".format(_type)) + record = Record.new( + zone, + name, + data_for(_type, record), + source=self, + lenient=lenient, + ) + zone.add_record(record, lenient=lenient) + + self.log.info( + "populate: found %s records, exists=%s", + len(zone.records) - before, + exists, + ) + return exists + + def _should_ignore(self, record): + name = record.get("name", "name-not-defined") + if record.get("filters") is None: + return False + want_filters = 3 + filters = record.get("filters", []) + if len(filters) != want_filters: + self.log.info( + "ignore %s has filters and their count is not %d", + name, + want_filters, + ) + return True + types = [v.get("type") for v in filters] + for i, want_type in enumerate(["geodns", "default", "first_n"]): + if types[i] != want_type: + self.log.info( + "ignore %s, filters.%d.type is %s, want %s", + name, + i, + types[i], + want_type, + ) + return True + limits = [filters[i].get("limit", 1) for i in [1, 2]] + if limits[0] != limits[1]: + self.log.info( + "ignore %s, filters.1.limit (%d) != filters.2.limit (%d)", + name, + limits[0], + limits[1], + ) + return True + return False + + def _params_for_dymanic(self, record): + records = [] + default_pool_found = False + default_values = set( + record.values if hasattr(record, "values") else [record.value] + ) + for rule in record.dynamic.rules: + meta = dict() + # build meta tags if geos information present + if len(rule.data.get("geos", [])) > 0: + for geo_code in rule.data["geos"]: + geo = GeoCodes.parse(geo_code) + + country = geo["country_code"] + continent = geo["continent_code"] + if country is not None: + meta.setdefault("countries", []).append(country) + else: + meta.setdefault("continents", []).append(continent) + else: + meta["default"] = True + + pool_values = set() + pool_name = rule.data["pool"] + for value in record.dynamic.pools[pool_name].data["values"]: + v = value["value"] + records.append({"content": [v], "meta": meta}) + pool_values.add(v) + + default_pool_found |= default_values == pool_values + + # if default values doesn't match any pool values, then just add this + # values with no any meta + if not default_pool_found: + for value in default_values: + records.append({"content": [value]}) + + return records + + def _params_for_single(self, record): + return { + "ttl": record.ttl, + "resource_records": [{"content": [record.value]}], + } + + _params_for_PTR = _params_for_single + + def _params_for_CNAME(self, record): + if not record.dynamic: + return self._params_for_single(record) + + return { + "ttl": record.ttl, + "resource_records": self._params_for_dymanic(record), + "filters": [ + {"type": "geodns"}, + { + "type": "default", + "limit": self.records_per_response, + "strict": False, + }, + {"type": "first_n", "limit": self.records_per_response}, + ], + } + + def _params_for_multiple(self, record): + extra = dict() + if record.dynamic: + extra["resource_records"] = self._params_for_dymanic(record) + extra["filters"] = [ + {"type": "geodns"}, + { + "type": "default", + "limit": self.records_per_response, + "strict": False, + }, + {"type": "first_n", "limit": self.records_per_response}, + ] + else: + extra["resource_records"] = [ + {"content": [value]} for value in record.values + ] + return { + "ttl": record.ttl, + **extra, + } + + _params_for_A = _params_for_multiple + _params_for_AAAA = _params_for_multiple + + def _params_for_NS(self, record): + return { + "ttl": record.ttl, + "resource_records": [ + {"content": [value]} for value in record.values + ], + } + + def _params_for_TXT(self, record): + return { + "ttl": record.ttl, + "resource_records": [ + {"content": [value.replace("\\;", ";")]} + for value in record.values + ], + } + + def _params_for_MX(self, record): + return { + "ttl": record.ttl, + "resource_records": [ + {"content": [rec.preference, rec.exchange]} + for rec in record.values + ], + } + + def _params_for_SRV(self, record): + return { + "ttl": record.ttl, + "resource_records": [ + {"content": [rec.priority, rec.weight, rec.port, rec.target]} + for rec in record.values + ], + } + + def _apply_create(self, change): + self.log.info("creating: %s", change) + new = change.new + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_create( + new.zone.name[:-1], new.fqdn, new._type, data + ) + + def _apply_update(self, change): + self.log.info("updating: %s", change) + new = change.new + data = getattr(self, "_params_for_{}".format(new._type))(new) + self._client.record_update( + new.zone.name[:-1], new.fqdn, new._type, data + ) + + def _apply_delete(self, change): + self.log.info("deleting: %s", change) + existing = change.existing + self._client.record_delete( + existing.zone.name[:-1], existing.fqdn, existing._type + ) + + def _apply(self, plan): + desired = plan.desired + changes = plan.changes + zone = desired.name[:-1] + self.log.debug( + "_apply: zone=%s, len(changes)=%d", desired.name, len(changes) + ) + + try: + self._client.zone(zone) + except GCoreClientNotFound: + self.log.info("_apply: no existing zone, trying to create it") + self._client.zone_create(zone) + self.log.info("_apply: zone has been successfully created") + + changes.reverse() + + for change in changes: + class_name = change.__class__.__name__ + getattr(self, "_apply_{}".format(class_name.lower()))(change) diff --git a/octodns/provider/mythicbeasts.py b/octodns/provider/mythicbeasts.py index b255a74..e1a2b04 100644 --- a/octodns/provider/mythicbeasts.py +++ b/octodns/provider/mythicbeasts.py @@ -70,13 +70,13 @@ class MythicBeastsProvider(BaseProvider): ... mythicbeasts: class: octodns.provider.mythicbeasts.MythicBeastsProvider - passwords: - my.domain.: 'password' + passwords: + my.domain.: 'DNS API v1 password' zones: my.domain.: targets: - - mythic + - mythicbeasts ''' RE_MX = re.compile(r'^(?P[0-9]+)\s+(?P\S+)$', diff --git a/octodns/provider/ns1.py b/octodns/provider/ns1.py index 87013fa..0ebb36b 100644 --- a/octodns/provider/ns1.py +++ b/octodns/provider/ns1.py @@ -77,8 +77,10 @@ class Ns1Client(object): self._datafeed = client.datafeed() self._datasource_id = None + self._feeds_for_monitors = None self._monitors_cache = None + self._notifylists_cache = None @property def datasource_id(self): @@ -121,6 +123,14 @@ class Ns1Client(object): {m['id']: m for m in self.monitors_list()} return self._monitors_cache + @property + def notifylists(self): + if self._notifylists_cache is None: + self.log.debug('notifylists: fetching & building') + self._notifylists_cache = \ + {l['name']: l for l in self.notifylists_list()} + return self._notifylists_cache + def datafeed_create(self, sourceid, name, config): ret = self._try(self._datafeed.create, sourceid, name, config) self.feeds_for_monitors[config['jobid']] = ret['id'] @@ -163,10 +173,17 @@ class Ns1Client(object): return ret def notifylists_delete(self, nlid): + for name, nl in self.notifylists.items(): + if nl['id'] == nlid: + del self._notifylists_cache[name] + break return self._try(self._notifylists.delete, nlid) def notifylists_create(self, **body): - return self._try(self._notifylists.create, body) + nl = self._try(self._notifylists.create, body) + # cache it + self.notifylists[nl['name']] = nl + return nl def notifylists_list(self): return self._try(self._notifylists.list) @@ -216,6 +233,13 @@ class Ns1Provider(BaseProvider): # Only required if using dynamic records monitor_regions: - lga + # Optional. Default: false. true is Recommended, but not the default + # for backwards compatibility reasons. If true, all NS1 monitors will + # use a shared notify list rather than one per record & value + # combination. See CHANGELOG, + # https://github.com/octodns/octodns/blob/master/CHANGELOG.md, for more + # information before enabling this behavior. + shared_notifylist: false # Optional. Default: None. If set, back off in advance to avoid 429s # from rate-limiting. Generally this should be set to the number # of processes or workers hitting the API, e.g. the value of @@ -237,6 +261,7 @@ class Ns1Provider(BaseProvider): 'NS', 'PTR', 'SPF', 'SRV', 'TXT', 'URLFWD')) ZONE_NOT_FOUND_MESSAGE = 'server error: zone not found' + SHARED_NOTIFYLIST_NAME = 'octoDNS NS1 Notify List' def _update_filter(self, filter, with_disabled): if with_disabled: @@ -368,7 +393,8 @@ class Ns1Provider(BaseProvider): } def __init__(self, id, api_key, retry_count=4, monitor_regions=None, - parallelism=None, client_config=None, *args, **kwargs): + parallelism=None, client_config=None, shared_notifylist=False, + *args, **kwargs): self.log = getLogger('Ns1Provider[{}]'.format(id)) self.log.debug('__init__: id=%s, api_key=***, retry_count=%d, ' 'monitor_regions=%s, parallelism=%s, client_config=%s', @@ -376,6 +402,7 @@ class Ns1Provider(BaseProvider): client_config) super(Ns1Provider, self).__init__(id, *args, **kwargs) self.monitor_regions = monitor_regions + self.shared_notifylist = shared_notifylist self._client = Ns1Client(api_key, parallelism, retry_count, client_config) @@ -888,7 +915,6 @@ class Ns1Provider(BaseProvider): def _feed_create(self, monitor): monitor_id = monitor['id'] self.log.debug('_feed_create: monitor=%s', monitor_id) - # TODO: looks like length limit is 64 char name = '{} - {}'.format(monitor['name'], self._uuid()[:6]) # Create the data feed @@ -902,22 +928,36 @@ class Ns1Provider(BaseProvider): return feed_id + def _notifylists_find_or_create(self, name): + self.log.debug('_notifylists_find_or_create: name="%s"', name) + try: + nl = self._client.notifylists[name] + self.log.debug('_notifylists_find_or_create: existing=%s', + nl['id']) + except KeyError: + notify_list = [{ + 'config': { + 'sourceid': self._client.datasource_id, + }, + 'type': 'datafeed', + }] + nl = self._client.notifylists_create(name=name, + notify_list=notify_list) + self.log.debug('_notifylists_find_or_create: created=%s', + nl['id']) + + return nl + def _monitor_create(self, monitor): self.log.debug('_monitor_create: monitor="%s"', monitor['name']) - # Create the notify list - notify_list = [{ - 'config': { - 'sourceid': self._client.datasource_id, - }, - 'type': 'datafeed', - }] - nl = self._client.notifylists_create(name=monitor['name'], - notify_list=notify_list) - nl_id = nl['id'] - self.log.debug('_monitor_create: notify_list=%s', nl_id) + + # Find the right notifylist + nl_name = self.SHARED_NOTIFYLIST_NAME \ + if self.shared_notifylist else monitor['name'] + nl = self._notifylists_find_or_create(nl_name) # Create the monitor - monitor['notify_list'] = nl_id + monitor['notify_list'] = nl['id'] monitor = self._client.monitors_create(**monitor) monitor_id = monitor['id'] self.log.debug('_monitor_create: monitor=%s', monitor_id) @@ -1028,7 +1068,13 @@ class Ns1Provider(BaseProvider): self._client.monitors_delete(monitor_id) notify_list_id = monitor['notify_list'] - self._client.notifylists_delete(notify_list_id) + for nl_name, nl in self._client.notifylists.items(): + if nl['id'] == notify_list_id: + # We've found the that might need deleting + if nl['name'] != self.SHARED_NOTIFYLIST_NAME: + # It's not shared so is safe to delete + self._client.notifylists_delete(notify_list_id) + break def _add_answers_for_pool(self, answers, default_answers, pool_name, pool_label, pool_answers, pools, priority): @@ -1364,7 +1410,9 @@ class Ns1Provider(BaseProvider): params, active_monitor_ids = \ getattr(self, '_params_for_{}'.format(_type))(new) self._client.records_update(zone, domain, _type, **params) - self._monitors_gc(new, active_monitor_ids) + # If we're cleaning up we need to send in the old record since it'd + # have anything that needs cleaning up + self._monitors_gc(change.existing, active_monitor_ids) def _apply_Delete(self, ns1_zone, change): existing = change.existing diff --git a/tests/fixtures/gcore-no-changes.json b/tests/fixtures/gcore-no-changes.json new file mode 100644 index 0000000..b1a3b25 --- /dev/null +++ b/tests/fixtures/gcore-no-changes.json @@ -0,0 +1,245 @@ +{ + "rrsets": [ + { + "name": "unit.tests", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "1.2.3.4" + ] + }, + { + "content": [ + "1.2.3.5" + ] + } + ] + }, + { + "name": "unit.tests", + "type": "NS", + "ttl": 300, + "resource_records": [ + { + "content": [ + "ns2.gcdn.services" + ] + }, + { + "content": [ + "ns1.gcorelabs.net" + ] + } + ] + }, + { + "name": "_imap._tcp", + "type": "SRV", + "ttl": 600, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } + ] + }, + { + "name": "_pop3._tcp", + "type": "SRV", + "ttl": 600, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } + ] + }, + { + "name": "_srv._tcp", + "type": "SRV", + "ttl": 600, + "resource_records": [ + { + "content": [ + 12, + 20, + 30, + "foo-2.unit.tests" + ] + }, + { + "content": [ + 10, + 20, + 30, + "foo-1.unit.tests" + ] + } + ] + }, + { + "name": "aaaa.unit.tests", + "type": "AAAA", + "ttl": 600, + "resource_records": [ + { + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + } + ] + }, + { + "name": "cname.unit.tests", + "type": "CNAME", + "ttl": 300, + "resource_records": [ + { + "content": [ + "unit.tests." + ] + } + ] + }, + { + "name": "excluded.unit.tests", + "type": "CNAME", + "ttl": 3600, + "resource_records": [ + { + "content": [ + "unit.tests." + ] + } + ] + }, + { + "name": "mx.unit.tests", + "type": "MX", + "ttl": 300, + "resource_records": [ + { + "content": [ + 40, + "smtp-1.unit.tests." + ] + }, + { + "content": [ + 20, + "smtp-2.unit.tests." + ] + }, + { + "content": [ + 30, + "smtp-3.unit.tests." + ] + }, + { + "content": [ + 10, + "smtp-4.unit.tests." + ] + } + ] + }, + { + "name": "ptr.unit.tests.", + "type": "PTR", + "ttl": 300, + "resource_records": [ + { + "content": [ + "foo.bar.com" + ] + } + ] + }, + { + "name": "sub.unit.tests", + "type": "NS", + "ttl": 3600, + "resource_records": [ + { + "content": [ + "6.2.3.4" + ] + }, + { + "content": [ + "7.2.3.4" + ] + } + ] + }, + { + "name": "txt.unit.tests", + "type": "TXT", + "ttl": 600, + "resource_records": [ + { + "content": [ + "Bah bah black sheep" + ] + }, + { + "content": [ + "have you any wool." + ] + }, + { + "content": [ + "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs" + ] + } + ] + }, + { + "name": "www.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + }, + { + "name": "www.sub.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + }, + { + "name": "spf.sub.unit.tests.", + "type": "SPF", + "ttl": 600, + "resource_records": [ + { + "content": [ + "v=spf1 ip4:192.168.0.1/16-all" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/gcore-records.json b/tests/fixtures/gcore-records.json new file mode 100644 index 0000000..9bf58d7 --- /dev/null +++ b/tests/fixtures/gcore-records.json @@ -0,0 +1,428 @@ +{ + "rrsets": [ + { + "name": "unit.tests", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "1.2.3.4" + ] + } + ] + }, + { + "name": "unit.tests", + "type": "NS", + "ttl": 300, + "resource_records": [ + { + "content": [ + "ns2.gcdn.services" + ] + }, + { + "content": [ + "ns1.gcorelabs.net" + ] + } + ] + }, + { + "name": "_imap._tcp", + "type": "SRV", + "ttl": 1200, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } + ] + }, + { + "name": "_pop3._tcp", + "type": "SRV", + "ttl": 1200, + "resource_records": [ + { + "content": [ + 0, + 0, + 0, + "." + ] + } + ] + }, + { + "name": "_srv._tcp", + "type": "SRV", + "ttl": 1200, + "resource_records": [ + { + "content": [ + 12, + 20, + 30, + "foo-2.unit.tests." + ] + }, + { + "content": [ + 10, + 20, + 30, + "foo-1.unit.tests." + ] + } + ] + }, + { + "name": "aaaa.unit.tests", + "type": "AAAA", + "ttl": 600, + "resource_records": [ + { + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + } + ] + }, + { + "name": "cname.unit.tests", + "type": "CNAME", + "ttl": 300, + "resource_records": [ + { + "content": [ + "unit.tests." + ] + } + ] + }, + { + "name": "mx.unit.tests", + "type": "MX", + "ttl": 600, + "resource_records": [ + { + "content": [ + 40, + "smtp-1.unit.tests." + ] + }, + { + "content": [ + 20, + "smtp-2.unit.tests." + ] + } + ] + }, + { + "name": "ptr.unit.tests.", + "type": "PTR", + "ttl": 300, + "resource_records": [ + { + "content": [ + "foo.bar.com" + ] + } + ] + }, + { + "name": "sub.unit.tests", + "type": "NS", + "ttl": 300, + "resource_records": [ + { + "content": [ + "6.2.3.4" + ] + }, + { + "content": [ + "7.2.3.4" + ] + } + ] + }, + { + "name": "txt.unit.tests", + "type": "TXT", + "ttl": 300, + "resource_records": [ + { + "content": [ + "\"Bah bah black sheep\"" + ] + }, + { + "content": [ + "\"have you any wool.\"" + ] + }, + { + "content": [ + "\"v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+of/long/string+with+numb3rs\"" + ] + } + ] + }, + { + "name": "www.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + }, + { + "name": "www.sub.unit.tests.", + "type": "A", + "ttl": 300, + "resource_records": [ + { + "content": [ + "2.2.3.6" + ] + } + ] + }, + { + "name": "geo-A-single.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "strict": false, + "type": "default" + }, + { + "limit": 1, + "type": "first_n" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "8.8.8.8" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "9.9.9.9" + ], + "meta": { + "continents": [ + "EU" + ] + } + }, + { + "content": [ + "10.10.10.10" + ], + "meta": { + "default": true + } + } + ] + }, + { + "name": "geo-no-def.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "strict": false, + "type": "default" + }, + { + "limit": 1, + "type": "first_n" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ], + "meta": { + "countries": [ + "RU" + ] + } + } + ] + }, + { + "name": "geo-CNAME.unit.tests.", + "type": "CNAME", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 1, + "strict": false, + "type": "default" + }, + { + "limit": 1, + "type": "first_n" + } + ], + "resource_records": [ + { + "content": [ + "ru-1.unit.tests" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "ru-2.unit.tests" + ], + "meta": { + "countries": [ + "RU" + ] + } + }, + { + "content": [ + "eu.unit.tests" + ], + "meta": { + "continents": [ + "EU" + ] + } + }, + { + "content": [ + "any.unit.tests." + ], + "meta": { + "default": true + } + } + ] + }, + { + "name": "geo-ignore-len-filters.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] + }, + { + "name": "geo-ignore-types.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodistance" + }, + { + "limit": 1, + "type": "first_n" + }, + { + "limit": 1, + "strict": false, + "type": "default" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] + }, + { + "name": "geo-ignore-limits.unit.tests.", + "type": "A", + "ttl": 300, + "filters": [ + { + "type": "geodns" + }, + { + "limit": 2, + "strict": false, + "type": "default" + }, + { + "limit": 1, + "type": "first_n" + } + ], + "resource_records": [ + { + "content": [ + "7.7.7.7" + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/gcore-zone.json b/tests/fixtures/gcore-zone.json new file mode 100644 index 0000000..925af72 --- /dev/null +++ b/tests/fixtures/gcore-zone.json @@ -0,0 +1,27 @@ +{ + "id": 27757, + "name": "unit.test", + "nx_ttl": 300, + "retry": 5400, + "refresh": 0, + "expiry": 1209600, + "contact": "support@gcorelabs.com", + "serial": 1614752868, + "primary_server": "ns1.gcorelabs.net", + "records": [ + { + "id": 12419, + "name": "unit.test", + "type": "ns", + "ttl": 300, + "short_answers": [ + "[ns2.gcdn.services]", + "[ns1.gcorelabs.net]" + ] + } + ], + "dns_servers": [ + "ns1.gcorelabs.net", + "ns2.gcdn.services" + ] +} \ No newline at end of file diff --git a/tests/test_octodns_provider_gcore.py b/tests/test_octodns_provider_gcore.py new file mode 100644 index 0000000..2151440 --- /dev/null +++ b/tests/test_octodns_provider_gcore.py @@ -0,0 +1,672 @@ +# +# +# + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from mock import Mock, call +from os.path import dirname, join +from requests_mock import ANY, mock as requests_mock +from six import text_type +from unittest import TestCase + +from octodns.record import Record, Update, Delete, Create +from octodns.provider.gcore import ( + GCoreProvider, + GCoreClientBadRequest, + GCoreClientNotFound, + GCoreClientException, +) +from octodns.provider.yaml import YamlProvider +from octodns.zone import Zone + + +class TestGCoreProvider(TestCase): + expected = Zone("unit.tests.", []) + source = YamlProvider("test", join(dirname(__file__), "config")) + source.populate(expected) + + default_filters = [ + {"type": "geodns"}, + { + "type": "default", + "limit": 1, + "strict": False, + }, + {"type": "first_n", "limit": 1}, + ] + + def test_populate(self): + + provider = GCoreProvider("test_id", token="token") + + # TC: 400 - Bad Request. + with requests_mock() as mock: + mock.get(ANY, status_code=400, text='{"error":"bad body"}') + + with self.assertRaises(GCoreClientBadRequest) as ctx: + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertIn('"error":"bad body"', text_type(ctx.exception)) + + # TC: 404 - Not Found. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + + with self.assertRaises(GCoreClientNotFound) as ctx: + zone = Zone("unit.tests.", []) + provider._client.zone(zone.name) + self.assertIn( + '"error":"zone is not found"', text_type(ctx.exception) + ) + + # TC: General error + with requests_mock() as mock: + mock.get(ANY, status_code=500, text="Things caught fire") + + with self.assertRaises(GCoreClientException) as ctx: + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEqual("Things caught fire", text_type(ctx.exception)) + + # TC: No credentials or token error + with requests_mock() as mock: + with self.assertRaises(ValueError) as ctx: + GCoreProvider("test_id") + self.assertEqual( + "either token or login & password must be set", + text_type(ctx.exception), + ) + + # TC: Auth with login password + with requests_mock() as mock: + + def match_body(request): + return {"username": "foo", "password": "bar"} == request.json() + + auth_url = "http://api/auth/jwt/login" + mock.post( + auth_url, + additional_matcher=match_body, + status_code=200, + json={"access": "access"}, + ) + + providerPassword = GCoreProvider( + "test_id", + url="http://dns", + auth_url="http://api", + login="foo", + password="bar", + ) + assert mock.called + + # make sure token passed in header + zone_rrset_url = "http://dns/zones/unit.tests/rrsets?all=true" + mock.get( + zone_rrset_url, + request_headers={"Authorization": "Bearer access"}, + status_code=404, + ) + zone = Zone("unit.tests.", []) + assert not providerPassword.populate(zone) + + # TC: No diffs == no changes + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + with open("tests/fixtures/gcore-no-changes.json") as fh: + mock.get(base, text=fh.read()) + + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEqual(14, len(zone.records)) + self.assertEqual( + { + "", + "_imap._tcp", + "_pop3._tcp", + "_srv._tcp", + "aaaa", + "cname", + "excluded", + "mx", + "ptr", + "sub", + "txt", + "www", + "www.sub", + }, + {r.name for r in zone.records}, + ) + changes = self.expected.changes(zone, provider) + self.assertEqual(0, len(changes)) + + # TC: 4 create (dynamic) + 1 removed + 7 modified + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + with open("tests/fixtures/gcore-records.json") as fh: + mock.get(base, text=fh.read()) + + zone = Zone("unit.tests.", []) + provider.populate(zone) + self.assertEqual(16, len(zone.records)) + changes = self.expected.changes(zone, provider) + self.assertEqual(11, len(changes)) + self.assertEqual( + 3, len([c for c in changes if isinstance(c, Create)]) + ) + self.assertEqual( + 1, len([c for c in changes if isinstance(c, Delete)]) + ) + self.assertEqual( + 7, len([c for c in changes if isinstance(c, Update)]) + ) + + # TC: no pools can be built + with requests_mock() as mock: + base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets" + mock.get( + base, + json={ + "rrsets": [ + { + "name": "unit.tests.", + "type": "A", + "ttl": 300, + "filters": self.default_filters, + "resource_records": [{"content": ["7.7.7.7"]}], + } + ] + }, + ) + + zone = Zone("unit.tests.", []) + with self.assertRaises(RuntimeError) as ctx: + provider.populate(zone) + + self.assertTrue( + str(ctx.exception).startswith( + "filter is enabled, but no pools where built for" + ), + "{} - is not start from desired text".format(ctx.exception), + ) + + def test_apply(self): + provider = GCoreProvider("test_id", url="http://api", token="token") + + # TC: Zone does not exists but can be created. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + mock.post(ANY, status_code=200, text='{"id":1234}') + + plan = provider.plan(self.expected) + provider.apply(plan) + + # TC: Zone does not exists and can't be created. + with requests_mock() as mock: + mock.get( + ANY, status_code=404, text='{"error":"zone is not found"}' + ) + mock.post( + ANY, + status_code=400, + text='{"error":"parent zone is already' + ' occupied by another client"}', + ) + + with self.assertRaises( + (GCoreClientNotFound, GCoreClientBadRequest) + ) as ctx: + plan = provider.plan(self.expected) + provider.apply(plan) + self.assertIn( + "parent zone is already occupied by another client", + text_type(ctx.exception), + ) + + resp = Mock() + resp.json = Mock() + provider._client._request = Mock(return_value=resp) + + with open("tests/fixtures/gcore-zone.json") as fh: + zone = fh.read() + + # non-existent domain + resp.json.side_effect = [ + GCoreClientNotFound(resp), # no zone in populate + GCoreClientNotFound(resp), # no domain during apply + zone, + ] + plan = provider.plan(self.expected) + + # TC: create all + self.assertEqual(13, len(plan.changes)) + self.assertEqual(13, provider.apply(plan)) + self.assertFalse(plan.exists) + + provider._client._request.assert_has_calls( + [ + call( + "GET", + "http://api/zones/unit.tests/rrsets", + params={"all": "true"}, + ), + call("GET", "http://api/zones/unit.tests"), + call("POST", "http://api/zones", data={"name": "unit.tests"}), + call( + "POST", + "http://api/zones/unit.tests/www.sub.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["2.2.3.6"]}], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/www.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["2.2.3.6"]}], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/txt.unit.tests./TXT", + data={ + "ttl": 600, + "resource_records": [ + {"content": ["Bah bah black sheep"]}, + {"content": ["have you any wool."]}, + { + "content": [ + "v=DKIM1;k=rsa;s=email;h=sha256;p=A/kinda+" + "of/long/string+with+numb3rs" + ] + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/sub.unit.tests./NS", + data={ + "ttl": 3600, + "resource_records": [ + {"content": ["6.2.3.4."]}, + {"content": ["7.2.3.4."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/ptr.unit.tests./PTR", + data={ + "ttl": 300, + "resource_records": [ + {"content": ["foo.bar.com."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/mx.unit.tests./MX", + data={ + "ttl": 300, + "resource_records": [ + {"content": [10, "smtp-4.unit.tests."]}, + {"content": [20, "smtp-2.unit.tests."]}, + {"content": [30, "smtp-3.unit.tests."]}, + {"content": [40, "smtp-1.unit.tests."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/excluded.unit.tests./CNAME", + data={ + "ttl": 3600, + "resource_records": [{"content": ["unit.tests."]}], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/cname.unit.tests./CNAME", + data={ + "ttl": 300, + "resource_records": [{"content": ["unit.tests."]}], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/aaaa.unit.tests./AAAA", + data={ + "ttl": 600, + "resource_records": [ + { + "content": [ + "2601:644:500:e210:62f8:1dff:feb8:947a" + ] + } + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/_srv._tcp.unit.tests./SRV", + data={ + "ttl": 600, + "resource_records": [ + {"content": [10, 20, 30, "foo-1.unit.tests."]}, + {"content": [12, 20, 30, "foo-2.unit.tests."]}, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/_pop3._tcp.unit.tests./SRV", + data={ + "ttl": 600, + "resource_records": [{"content": [0, 0, 0, "."]}], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/_imap._tcp.unit.tests./SRV", + data={ + "ttl": 600, + "resource_records": [{"content": [0, 0, 0, "."]}], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/unit.tests./A", + data={ + "ttl": 300, + "resource_records": [ + {"content": ["1.2.3.4"]}, + {"content": ["1.2.3.5"]}, + ], + }, + ), + ] + ) + # expected number of total calls + self.assertEqual(16, provider._client._request.call_count) + + # TC: delete 1 and update 1 + provider._client._request.reset_mock() + provider._client.zone_records = Mock( + return_value=[ + { + "name": "www", + "ttl": 300, + "type": "A", + "resource_records": [{"content": ["1.2.3.4"]}], + }, + { + "name": "ttl", + "ttl": 600, + "type": "A", + "resource_records": [{"content": ["3.2.3.4"]}], + }, + ] + ) + + # Domain exists, we don't care about return + resp.json.side_effect = ["{}"] + + wanted = Zone("unit.tests.", []) + wanted.add_record( + Record.new( + wanted, "ttl", {"ttl": 300, "type": "A", "value": "3.2.3.4"} + ) + ) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEqual(2, len(plan.changes)) + self.assertEqual(2, provider.apply(plan)) + + provider._client._request.assert_has_calls( + [ + call( + "DELETE", "http://api/zones/unit.tests/www.unit.tests./A" + ), + call( + "PUT", + "http://api/zones/unit.tests/ttl.unit.tests./A", + data={ + "ttl": 300, + "resource_records": [{"content": ["3.2.3.4"]}], + }, + ), + ] + ) + + # TC: create dynamics + provider._client._request.reset_mock() + provider._client.zone_records = Mock(return_value=[]) + + # Domain exists, we don't care about return + resp.json.side_effect = ["{}"] + + wanted = Zone("unit.tests.", []) + wanted.add_record( + Record.new( + wanted, + "geo-simple", + { + "ttl": 300, + "type": "A", + "value": "3.3.3.3", + "dynamic": { + "pools": { + "pool-1": { + "fallback": "other", + "values": [ + {"value": "1.1.1.1"}, + {"value": "1.1.1.2"}, + ], + }, + "pool-2": { + "fallback": "other", + "values": [ + {"value": "2.2.2.1"}, + ], + }, + "other": {"values": [{"value": "3.3.3.3"}]}, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU-RU"]}, + {"pool": "pool-2", "geos": ["EU"]}, + {"pool": "other"}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "geo-defaults", + { + "ttl": 300, + "type": "A", + "value": "3.2.3.4", + "dynamic": { + "pools": { + "pool-1": { + "values": [ + {"value": "2.2.2.1"}, + ], + }, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU"]}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "cname-smpl", + { + "ttl": 300, + "type": "CNAME", + "value": "en.unit.tests.", + "dynamic": { + "pools": { + "pool-1": { + "fallback": "other", + "values": [ + {"value": "ru-1.unit.tests."}, + {"value": "ru-2.unit.tests."}, + ], + }, + "pool-2": { + "fallback": "other", + "values": [ + {"value": "eu.unit.tests."}, + ], + }, + "other": {"values": [{"value": "en.unit.tests."}]}, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU-RU"]}, + {"pool": "pool-2", "geos": ["EU"]}, + {"pool": "other"}, + ], + }, + }, + ), + ) + wanted.add_record( + Record.new( + wanted, + "cname-dflt", + { + "ttl": 300, + "type": "CNAME", + "value": "en.unit.tests.", + "dynamic": { + "pools": { + "pool-1": { + "values": [ + {"value": "eu.unit.tests."}, + ], + }, + }, + "rules": [ + {"pool": "pool-1", "geos": ["EU"]}, + ], + }, + }, + ), + ) + + plan = provider.plan(wanted) + self.assertTrue(plan.exists) + self.assertEqual(4, len(plan.changes)) + self.assertEqual(4, provider.apply(plan)) + + provider._client._request.assert_has_calls( + [ + call( + "POST", + "http://api/zones/unit.tests/geo-simple.unit.tests./A", + data={ + "ttl": 300, + "filters": self.default_filters, + "resource_records": [ + { + "content": ["1.1.1.1"], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["1.1.1.2"], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["2.2.2.1"], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["3.3.3.3"], + "meta": {"default": True}, + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/geo-defaults.unit.tests./A", + data={ + "ttl": 300, + "filters": self.default_filters, + "resource_records": [ + { + "content": ["2.2.2.1"], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["3.2.3.4"], + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/cname-smpl.unit.tests./CNAME", + data={ + "ttl": 300, + "filters": self.default_filters, + "resource_records": [ + { + "content": ["ru-1.unit.tests."], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["ru-2.unit.tests."], + "meta": {"countries": ["RU"]}, + }, + { + "content": ["eu.unit.tests."], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["en.unit.tests."], + "meta": {"default": True}, + }, + ], + }, + ), + call( + "POST", + "http://api/zones/unit.tests/cname-dflt.unit.tests./CNAME", + data={ + "ttl": 300, + "filters": self.default_filters, + "resource_records": [ + { + "content": ["eu.unit.tests."], + "meta": {"continents": ["EU"]}, + }, + { + "content": ["en.unit.tests."], + }, + ], + }, + ), + ] + ) diff --git a/tests/test_octodns_provider_ns1.py b/tests/test_octodns_provider_ns1.py index da7f3de..932e4c4 100644 --- a/tests/test_octodns_provider_ns1.py +++ b/tests/test_octodns_provider_ns1.py @@ -768,12 +768,70 @@ class TestNs1ProviderDynamic(TestCase): monitor = { 'name': 'test monitor', } + provider._client._notifylists_cache = {} monitor_id, feed_id = provider._monitor_create(monitor) self.assertEquals('mon-id', monitor_id) self.assertEquals('feed-id', feed_id) monitors_create_mock.assert_has_calls([call(name='test monitor', notify_list='nl-id')]) + @patch('octodns.provider.ns1.Ns1Provider._feed_create') + @patch('octodns.provider.ns1.Ns1Client.monitors_create') + @patch('octodns.provider.ns1.Ns1Client._try') + def test_monitor_create_shared_notifylist(self, try_mock, + monitors_create_mock, + feed_create_mock): + provider = Ns1Provider('test', 'api-key', shared_notifylist=True) + + # pre-fill caches to avoid extranious calls (things we're testing + # elsewhere) + provider._client._datasource_id = 'foo' + provider._client._feeds_for_monitors = {} + + # First time we'll need to create the share list + provider._client._notifylists_cache = {} + try_mock.reset_mock() + monitors_create_mock.reset_mock() + feed_create_mock.reset_mock() + try_mock.side_effect = [{ + 'id': 'nl-id', + 'name': provider.SHARED_NOTIFYLIST_NAME, + }] + monitors_create_mock.side_effect = [{ + 'id': 'mon-id', + }] + feed_create_mock.side_effect = ['feed-id'] + monitor = { + 'name': 'test monitor', + } + monitor_id, feed_id = provider._monitor_create(monitor) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitors_create_mock.assert_has_calls([call(name='test monitor', + notify_list='nl-id')]) + try_mock.assert_called_once() + # The shared notifylist should be cached now + self.assertEquals([provider.SHARED_NOTIFYLIST_NAME], + list(provider._client._notifylists_cache.keys())) + + # Second time we'll use the cached version + try_mock.reset_mock() + monitors_create_mock.reset_mock() + feed_create_mock.reset_mock() + monitors_create_mock.side_effect = [{ + 'id': 'mon-id', + }] + feed_create_mock.side_effect = ['feed-id'] + monitor = { + 'name': 'test monitor', + } + monitor_id, feed_id = provider._monitor_create(monitor) + self.assertEquals('mon-id', monitor_id) + self.assertEquals('feed-id', feed_id) + monitors_create_mock.assert_has_calls([call(name='test monitor', + notify_list='nl-id')]) + try_mock.assert_not_called() + def test_monitor_gen(self): provider = Ns1Provider('test', 'api-key') @@ -986,6 +1044,12 @@ class TestNs1ProviderDynamic(TestCase): 'notify_list': 'nl-id', } }] + provider._client._notifylists_cache = { + 'not shared': { + 'id': 'nl-id', + 'name': 'not shared', + } + } provider._monitors_gc(record) monitors_for_mock.assert_has_calls([call(record)]) datafeed_delete_mock.assert_has_calls([call('foo', 'feed-id')]) @@ -1025,12 +1089,75 @@ class TestNs1ProviderDynamic(TestCase): 'notify_list': 'nl-id2', }, }] + provider._client._notifylists_cache = { + 'not shared': { + 'id': 'nl-id', + 'name': 'not shared', + }, + 'not shared 2': { + 'id': 'nl-id2', + 'name': 'not shared 2', + } + } provider._monitors_gc(record, {'mon-id'}) monitors_for_mock.assert_has_calls([call(record)]) datafeed_delete_mock.assert_not_called() monitors_delete_mock.assert_has_calls([call('mon-id2')]) notifylists_delete_mock.assert_has_calls([call('nl-id2')]) + # Non-active monitor w/o a notifylist, generally shouldn't happen, but + # code should handle it just in case someone gets clicky in the UI + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'y': { + 'id': 'mon-id2', + 'notify_list': 'nl-id2', + }, + }] + provider._client._notifylists_cache = { + 'not shared a': { + 'id': 'nl-ida', + 'name': 'not shared a', + }, + 'not shared b': { + 'id': 'nl-idb', + 'name': 'not shared b', + } + } + provider._monitors_gc(record, {'mon-id'}) + monitors_for_mock.assert_has_calls([call(record)]) + datafeed_delete_mock.assert_not_called() + monitors_delete_mock.assert_has_calls([call('mon-id2')]) + notifylists_delete_mock.assert_not_called() + + # Non-active monitor with a shared notifylist, monitor deleted, but + # notifylist is left alone + provider.shared_notifylist = True + monitors_for_mock.reset_mock() + datafeed_delete_mock.reset_mock() + monitors_delete_mock.reset_mock() + notifylists_delete_mock.reset_mock() + monitors_for_mock.side_effect = [{ + 'y': { + 'id': 'mon-id2', + 'notify_list': 'shared', + }, + }] + provider._client._notifylists_cache = { + 'shared': { + 'id': 'shared', + 'name': provider.SHARED_NOTIFYLIST_NAME, + }, + } + provider._monitors_gc(record, {'mon-id'}) + monitors_for_mock.assert_has_calls([call(record)]) + datafeed_delete_mock.assert_not_called() + monitors_delete_mock.assert_has_calls([call('mon-id2')]) + notifylists_delete_mock.assert_not_called() + @patch('octodns.provider.ns1.Ns1Provider._monitor_sync') @patch('octodns.provider.ns1.Ns1Provider._monitors_for') def test_params_for_dynamic_region_only(self, monitors_for_mock, @@ -2329,17 +2456,22 @@ class TestNs1Client(TestCase): notifylists_list_mock.reset_mock() notifylists_create_mock.reset_mock() notifylists_delete_mock.reset_mock() - notifylists_create_mock.side_effect = ['bar'] + notifylists_list_mock.side_effect = [{}] + expected = { + 'id': 'nl-id', + 'name': 'bar', + } + notifylists_create_mock.side_effect = [expected] notify_list = [{ 'config': { 'sourceid': 'foo', }, 'type': 'datafeed', }] - nl = client.notifylists_create(name='some name', - notify_list=notify_list) - self.assertEquals('bar', nl) - notifylists_list_mock.assert_not_called() + got = client.notifylists_create(name='some name', + notify_list=notify_list) + self.assertEquals(expected, got) + notifylists_list_mock.assert_called_once() notifylists_create_mock.assert_has_calls([ call({'name': 'some name', 'notify_list': notify_list}) ]) @@ -2353,6 +2485,29 @@ class TestNs1Client(TestCase): notifylists_create_mock.assert_not_called() notifylists_delete_mock.assert_has_calls([call('nlid')]) + # Delete again, this time with a cache item that needs cleaned out and + # another that needs to be ignored + notifylists_list_mock.reset_mock() + notifylists_create_mock.reset_mock() + notifylists_delete_mock.reset_mock() + client._notifylists_cache = { + 'another': { + 'id': 'notid', + 'name': 'another', + }, + # This one comes 2nd on purpose + 'the-one': { + 'id': 'nlid', + 'name': 'the-one', + }, + } + client.notifylists_delete('nlid') + notifylists_list_mock.assert_not_called() + notifylists_create_mock.assert_not_called() + notifylists_delete_mock.assert_has_calls([call('nlid')]) + # Only another left + self.assertEquals(['another'], list(client._notifylists_cache.keys())) + notifylists_list_mock.reset_mock() notifylists_create_mock.reset_mock() notifylists_delete_mock.reset_mock()