mirror/github-octodns
1
0
Fork 0
mirror of https://github.com/github/octodns.git synced 2024-05-11 05:55:00 +00:00
Code Issues Releases Activity

add support for dynamic A, AAAA, CNAME records

Browse Source
This commit is contained in:
Yaroshevich, Denis
2021-08-02 18:47:11 +03:00
parent cc0faf5762
commit 006a61e4d8
4 changed files with 764 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -204,7 +204,7 @@ The above command pulled the existing data out of Route53 and placed the results
| [EtcHostsProvider](/octodns/provider/etc_hosts.py) | | A, AAAA, ALIAS, CNAME | No | |
| [EnvVarSource](/octodns/source/envvar.py) | | TXT | No | read-only environment variable injection |
| [GandiProvider](/octodns/provider/gandi.py) | | A, AAAA, ALIAS, CAA, CNAME, DNAME, MX, NS, PTR, SPF, SRV, SSHFP, TXT | No | |
| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA, NS, MX, TXT, SRV, CNAME, PTR | No | |
| [GCoreProvider](/octodns/provider/gcore.py) | | A, AAAA, NS, MX, TXT, SRV, CNAME, PTR | Dynamic | |
| [GoogleCloudProvider](/octodns/provider/googlecloud.py) | google-cloud-dns | A, AAAA, CAA, CNAME, MX, NAPTR, NS, PTR, SPF, SRV, TXT | No | |
| [HetznerProvider](/octodns/provider/hetzner.py) | | A, AAAA, CAA, CNAME, MX, NS, SRV, TXT | No | |
| [MythicBeastsProvider](/octodns/provider/mythicbeasts.py) | Mythic Beasts | A, AAAA, ALIAS, CNAME, MX, NS, SRV, SSHFP, CAA, TXT | No | |

248
octodns/provider/gcore.py
View File

@@ -15,6 +15,7 @@ import http
import logging
import urllib.parse
from ..record import GeoCodes
from ..record import Record
from .base import BaseProvider
@@ -157,10 +158,11 @@ class GCoreProvider(BaseProvider):
password: XXXXXXXXXXXX
# auth_url: https://api.gcdn.co
# url: https://dnsapi.gcorelabs.com/v2
# records_per_response: 1
"""
SUPPORTS_GEO = False
SUPPORTS_DYNAMIC = False
SUPPORTS_DYNAMIC = True
SUPPORTS = set(("A", "AAAA", "NS", "MX", "TXT", "SRV", "CNAME", "PTR"))
def __init__(self, id, *args, **kwargs):
@@ -170,6 +172,7 @@ class GCoreProvider(BaseProvider):
password = kwargs.pop("password", None)
api_url = kwargs.pop("url", "https://dnsapi.gcorelabs.com/v2")
auth_url = kwargs.pop("auth_url", "https://api.gcdn.co")
self.records_per_response = kwargs.pop("records_per_response", 1)
self.log = logging.getLogger("GCoreProvider[{}]".format(id))
self.log.debug("__init__: id=%s", id)
super(GCoreProvider, self).__init__(id, *args, **kwargs)
@@ -186,6 +189,86 @@ class GCoreProvider(BaseProvider):
def _add_dot_if_need(self, value):
return "{}.".format(value) if not value.endswith(".") else value
def _build_pools(self, record, default_pool_name, value_transform_fn):
defaults = []
geo_sets, pool_idx = dict(), 0
pools = defaultdict(lambda: {"values": []})
for rr in record["resource_records"]:
meta = rr.get("meta", {})
value = {"value": value_transform_fn(rr["content"][0])}
countries = meta.get("countries", [])
continents = meta.get("continents", [])
if meta.get("default", False):
pools[default_pool_name]["values"].append(value)
defaults.append(value["value"])
continue
# defaults is false or missing and no conties or continents
elif len(continents) == 0 and len(countries) == 0:
defaults.append(value["value"])
continue
# RR with the same set of countries and continents are
# combined in single pool
geo_set = frozenset(
[GeoCodes.country_to_code(cc.upper()) for cc in countries]
) | frozenset(cc.upper() for cc in continents)
if geo_set not in geo_sets:
geo_sets[geo_set] = "pool-{}".format(pool_idx)
pool_idx += 1
pools[geo_sets[geo_set]]["values"].append(value)
return pools, geo_sets, defaults
def _build_rules(self, pools, geo_sets):
rules = []
for name, _ in pools.items():
rule = {"pool": name}
geo_set = next(
(
geo_set
for geo_set, pool_name in geo_sets.items()
if pool_name == name
),
{},
)
if len(geo_set) > 0:
rule["geos"] = list(geo_set)
rules.append(rule)
return sorted(rules, key=lambda x: x["pool"])
def _data_for_dynamic(self, record, value_transform_fn=lambda x: x):
default_pool = "other"
pools, geo_sets, defaults = self._build_pools(
record, default_pool, value_transform_fn
)
if len(pools) == 0:
raise RuntimeError(
"filter is enabled, but no pools where built for {}".format(
record
)
)
# defaults can't be empty, so use first pool values
if len(defaults) == 0:
defaults = [
value_transform_fn(v["value"])
for v in next(iter(pools.values()))["values"]
]
# if at least one default RR was found then setup fallback for
# other pools to default
if default_pool in pools:
for pool_name, pool in pools.items():
if pool_name == default_pool:
continue
pool["fallback"] = default_pool
rules = self._build_rules(pools, geo_sets)
return pools, rules, defaults
def _data_for_single(self, _type, record):
return {
"ttl": record["ttl"],
@@ -195,18 +278,42 @@ class GCoreProvider(BaseProvider):
),
}
_data_for_CNAME = _data_for_single
_data_for_PTR = _data_for_single
def _data_for_multiple(self, _type, record):
def _data_for_CNAME(self, _type, record):
if record.get("filters") is None:
return self._data_for_single(_type, record)
pools, rules, defaults = self._data_for_dynamic(
record, self._add_dot_if_need
)
return {
"ttl": record["ttl"],
"type": _type,
"values": [
rr_value
for resource_record in record["resource_records"]
for rr_value in resource_record["content"]
],
"dynamic": {"pools": pools, "rules": rules},
"value": self._add_dot_if_need(defaults[0]),
}
def _data_for_multiple(self, _type, record):
extra = dict()
if record.get("filters") is not None:
pools, rules, defaults = self._data_for_dynamic(record)
extra = {
"dynamic": {"pools": pools, "rules": rules},
"values": defaults,
}
else:
extra = {
"values": [
rr_value
for resource_record in record["resource_records"]
for rr_value in resource_record["content"]
]
}
return {
"ttl": record["ttl"],
"type": _type,
**extra,
}
_data_for_A = _data_for_multiple
@@ -286,6 +393,8 @@ class GCoreProvider(BaseProvider):
_type = record["type"].upper()
if _type not in self.SUPPORTS:
continue
if self._should_ignore(record):
continue
rr_name = zone.hostname_from_fqdn(record["name"])
values[rr_name][_type] = record
@@ -309,16 +418,132 @@ class GCoreProvider(BaseProvider):
)
return exists
def _should_ignore(self, record):
name = record.get("name", "name-not-defined")
if record.get("filters") is None:
return False
want_filters = 3
filters = record.get("filters", [])
if len(filters) != want_filters:
self.log.info(
"ignore %s has filters and their count is not %d",
name,
want_filters,
)
return True
types = [v.get("type") for v in filters]
for i, want_type in enumerate(["geodns", "first_n", "default"]):
if types[i] != want_type:
self.log.info(
"ignore %s, filters.%d.type is %s, want %s",
name,
i,
types[i],
want_type,
)
return True
limits = [filters[i].get("limit", 1) for i in [1, 2]]
if limits[0] != limits[1]:
self.log.info(
"ignore %s, filters.1.limit (%d) != filters.2.limit (%d)",
name,
limits[0],
limits[1],
)
return True
return False
def _params_for_dymanic(self, record):
records = []
default_pool_found = False
default_values = set(
record.values if hasattr(record, "values") else [record.value]
)
for rule in record.dynamic.rules:
meta = dict()
# build meta tags if geos information present
if len(rule.data.get("geos", [])) > 0:
for geo_code in rule.data["geos"]:
geo = GeoCodes.parse(geo_code)
country = geo["country_code"]
continent = geo["continent_code"]
if country is not None:
meta.setdefault("countries", []).append(country)
else:
meta.setdefault("continents", []).append(continent)
else:
meta["default"] = True
pool_values = set()
pool_name = rule.data["pool"]
for value in record.dynamic.pools[pool_name].data["values"]:
v = value["value"]
records.append({"content": [v], "meta": meta})
pool_values.add(v)
default_pool_found |= default_values == pool_values
# if default values doesn't match any pool values, then just add this
# values with no any meta
if not default_pool_found:
for value in default_values:
records.append({"content": [value]})
return records
def _params_for_single(self, record):
return {
"ttl": record.ttl,
"resource_records": [{"content": [record.value]}],
}
_params_for_CNAME = _params_for_single
_params_for_PTR = _params_for_single
def _params_for_CNAME(self, record):
if not record.dynamic:
return self._params_for_single(record)
return {
"ttl": record.ttl,
"resource_records": self._params_for_dymanic(record),
"filters": [
{"type": "geodns"},
{"type": "first_n", "limit": self.records_per_response},
{
"type": "default",
"limit": self.records_per_response,
"strict": False,
},
],
}
def _params_for_multiple(self, record):
extra = dict()
if record.dynamic:
extra["resource_records"] = self._params_for_dymanic(record)
extra["filters"] = [
{"type": "geodns"},
{"type": "first_n", "limit": self.records_per_response},
{
"type": "default",
"limit": self.records_per_response,
"strict": False,
},
]
else:
extra["resource_records"] = [
{"content": [value]} for value in record.values
]
return {
"ttl": record.ttl,
**extra,
}
_params_for_A = _params_for_multiple
_params_for_AAAA = _params_for_multiple
def _params_for_NS(self, record):
return {
"ttl": record.ttl,
"resource_records": [
@@ -326,12 +551,7 @@ class GCoreProvider(BaseProvider):
],
}
_params_for_A = _params_for_multiple
_params_for_AAAA = _params_for_multiple
_params_for_NS = _params_for_multiple
def _params_for_TXT(self, record):
# print(record.values)
return {
"ttl": record.ttl,
"resource_records": [

224
tests/fixtures/gcore-records.json vendored
View File

@@ -199,6 +199,230 @@
]
}
]
},
{
"name": "geo-A-single.unit.tests.",
"type": "A",
"ttl": 300,
"filters": [
{
"type": "geodns"
},
{
"limit": 1,
"type": "first_n"
},
{
"limit": 1,
"strict": false,
"type": "default"
}
],
"resource_records": [
{
"content": [
"7.7.7.7"
],
"meta": {
"countries": [
"RU"
]
}
},
{
"content": [
"8.8.8.8"
],
"meta": {
"countries": [
"RU"
]
}
},
{
"content": [
"9.9.9.9"
],
"meta": {
"continents": [
"EU"
]
}
},
{
"content": [
"10.10.10.10"
],
"meta": {
"default": true
}
}
]
},
{
"name": "geo-no-def.unit.tests.",
"type": "A",
"ttl": 300,
"filters": [
{
"type": "geodns"
},
{
"limit": 1,
"type": "first_n"
},
{
"limit": 1,
"strict": false,
"type": "default"
}
],
"resource_records": [
{
"content": [
"7.7.7.7"
],
"meta": {
"countries": [
"RU"
]
}
}
]
},
{
"name": "geo-CNAME.unit.tests.",
"type": "CNAME",
"ttl": 300,
"filters": [
{
"type": "geodns"
},
{
"limit": 1,
"type": "first_n"
},
{
"limit": 1,
"strict": false,
"type": "default"
}
],
"resource_records": [
{
"content": [
"ru-1.unit.tests"
],
"meta": {
"countries": [
"RU"
]
}
},
{
"content": [
"ru-2.unit.tests"
],
"meta": {
"countries": [
"RU"
]
}
},
{
"content": [
"eu.unit.tests"
],
"meta": {
"continents": [
"EU"
]
}
},
{
"content": [
"any.unit.tests."
],
"meta": {
"default": true
}
}
]
},
{
"name": "geo-ignore-len-filters.unit.tests.",
"type": "A",
"ttl": 300,
"filters": [
{
"limit": 1,
"type": "first_n"
},
{
"limit": 1,
"strict": false,
"type": "default"
}
],
"resource_records": [
{
"content": [
"7.7.7.7"
]
}
]
},
{
"name": "geo-ignore-types.unit.tests.",
"type": "A",
"ttl": 300,
"filters": [
{
"type": "geodistance"
},
{
"limit": 1,
"type": "first_n"
},
{
"limit": 1,
"strict": false,
"type": "default"
}
],
"resource_records": [
{
"content": [
"7.7.7.7"
]
}
]
},
{
"name": "geo-ignore-limits.unit.tests.",
"type": "A",
"ttl": 300,
"filters": [
{
"type": "geodns"
},
{
"limit": 1,
"type": "first_n"
},
{
"limit": 2,
"strict": false,
"type": "default"
}
],
"resource_records": [
{
"content": [
"7.7.7.7"
]
}
]
}
]
}

320
tests/test_octodns_provider_gcore.py
View File

@@ -15,7 +15,7 @@ from requests_mock import ANY, mock as requests_mock
from six import text_type
from unittest import TestCase
from octodns.record import Record, Update, Delete
from octodns.record import Record, Update, Delete, Create
from octodns.provider.gcore import (
GCoreProvider,
GCoreClientBadRequest,
@@ -35,7 +35,7 @@ class TestGCoreProvider(TestCase):
provider = GCoreProvider("test_id", token="token")
# 400 - Bad Request.
# TC: 400 - Bad Request.
with requests_mock() as mock:
mock.get(ANY, status_code=400, text='{"error":"bad body"}')
@@ -44,7 +44,7 @@ class TestGCoreProvider(TestCase):
provider.populate(zone)
self.assertIn('"error":"bad body"', text_type(ctx.exception))
# 404 - Not Found.
# TC: 404 - Not Found.
with requests_mock() as mock:
mock.get(
ANY, status_code=404, text='{"error":"zone is not found"}'
@@ -57,7 +57,7 @@ class TestGCoreProvider(TestCase):
'"error":"zone is not found"', text_type(ctx.exception)
)
# General error
# TC: General error
with requests_mock() as mock:
mock.get(ANY, status_code=500, text="Things caught fire")
@@ -66,7 +66,7 @@ class TestGCoreProvider(TestCase):
provider.populate(zone)
self.assertEqual("Things caught fire", text_type(ctx.exception))
# No credentials or token error
# TC: No credentials or token error
with requests_mock() as mock:
with self.assertRaises(ValueError) as ctx:
GCoreProvider("test_id")
@@ -75,7 +75,7 @@ class TestGCoreProvider(TestCase):
text_type(ctx.exception),
)
# Auth with login password
# TC: Auth with login password
with requests_mock() as mock:
def match_body(request):
@@ -108,7 +108,7 @@ class TestGCoreProvider(TestCase):
zone = Zone("unit.tests.", [])
assert not providerPassword.populate(zone)
# No diffs == no changes
# TC: No diffs == no changes
with requests_mock() as mock:
base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets"
with open("tests/fixtures/gcore-no-changes.json") as fh:
@@ -138,7 +138,7 @@ class TestGCoreProvider(TestCase):
changes = self.expected.changes(zone, provider)
self.assertEqual(0, len(changes))
# 1 removed + 7 modified
# TC: 4 create (dynamic) + 1 removed + 7 modified
with requests_mock() as mock:
base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets"
with open("tests/fixtures/gcore-records.json") as fh:
@@ -146,9 +146,12 @@ class TestGCoreProvider(TestCase):
zone = Zone("unit.tests.", [])
provider.populate(zone)
self.assertEqual(13, len(zone.records))
self.assertEqual(16, len(zone.records))
changes = self.expected.changes(zone, provider)
self.assertEqual(8, len(changes))
self.assertEqual(11, len(changes))
self.assertEqual(
3, len([c for c in changes if isinstance(c, Create)])
)
self.assertEqual(
1, len([c for c in changes if isinstance(c, Delete)])
)
@@ -156,10 +159,47 @@ class TestGCoreProvider(TestCase):
7, len([c for c in changes if isinstance(c, Update)])
)
# TC: no pools can be built
with requests_mock() as mock:
base = "https://dnsapi.gcorelabs.com/v2/zones/unit.tests/rrsets"
mock.get(
base,
json={
"rrsets": [
{
"name": "unit.tests.",
"type": "A",
"ttl": 300,
"filters": [
{"type": "geodns"},
{"limit": 1, "type": "first_n"},
{
"limit": 1,
"strict": False,
"type": "default",
},
],
"resource_records": [{"content": ["7.7.7.7"]}],
}
]
},
)
zone = Zone("unit.tests.", [])
with self.assertRaises(RuntimeError) as ctx:
provider.populate(zone)
self.assertTrue(
str(ctx.exception).startswith(
"filter is enabled, but no pools where built for"
),
"{} - is not start from desired text".format(ctx.exception),
)
def test_apply(self):
provider = GCoreProvider("test_id", url="http://api", token="token")
# Zone does not exists but can be created.
# TC: Zone does not exists but can be created.
with requests_mock() as mock:
mock.get(
ANY, status_code=404, text='{"error":"zone is not found"}'
@@ -169,7 +209,7 @@ class TestGCoreProvider(TestCase):
plan = provider.plan(self.expected)
provider.apply(plan)
# Zone does not exists and can't be created.
# TC: Zone does not exists and can't be created.
with requests_mock() as mock:
mock.get(
ANY, status_code=404, text='{"error":"zone is not found"}'
@@ -206,7 +246,7 @@ class TestGCoreProvider(TestCase):
]
plan = provider.plan(self.expected)
# create all
# TC: create all
self.assertEqual(13, len(plan.changes))
self.assertEqual(13, provider.apply(plan))
self.assertFalse(plan.exists)
@@ -360,9 +400,8 @@ class TestGCoreProvider(TestCase):
# expected number of total calls
self.assertEqual(16, provider._client._request.call_count)
# TC: delete 1 and update 1
provider._client._request.reset_mock()
# delete 1 and update 1
provider._client.zone_records = Mock(
return_value=[
{
@@ -410,3 +449,254 @@ class TestGCoreProvider(TestCase):
),
]
)
# TC: create dynamics
provider._client._request.reset_mock()
provider._client.zone_records = Mock(return_value=[])
# Domain exists, we don't care about return
resp.json.side_effect = ["{}"]
wanted = Zone("unit.tests.", [])
wanted.add_record(
Record.new(
wanted,
"geo-simple",
{
"ttl": 300,
"type": "A",
"value": "3.3.3.3",
"dynamic": {
"pools": {
"pool-1": {
"fallback": "other",
"values": [
{"value": "1.1.1.1"},
{"value": "1.1.1.2"},
],
},
"pool-2": {
"fallback": "other",
"values": [
{"value": "2.2.2.1"},
],
},
"other": {"values": [{"value": "3.3.3.3"}]},
},
"rules": [
{"pool": "pool-1", "geos": ["EU-RU"]},
{"pool": "pool-2", "geos": ["EU"]},
{"pool": "other"},
],
},
},
),
)
wanted.add_record(
Record.new(
wanted,
"geo-defaults",
{
"ttl": 300,
"type": "A",
"value": "3.2.3.4",
"dynamic": {
"pools": {
"pool-1": {
"values": [
{"value": "2.2.2.1"},
],
},
},
"rules": [
{"pool": "pool-1", "geos": ["EU"]},
],
},
},
),
)
wanted.add_record(
Record.new(
wanted,
"geo-cname-simple",
{
"ttl": 300,
"type": "CNAME",
"value": "en.unit.tests.",
"dynamic": {
"pools": {
"pool-1": {
"fallback": "other",
"values": [
{"value": "ru-1.unit.tests."},
{"value": "ru-2.unit.tests."},
],
},
"pool-2": {
"fallback": "other",
"values": [
{"value": "eu.unit.tests."},
],
},
"other": {"values": [{"value": "en.unit.tests."}]},
},
"rules": [
{"pool": "pool-1", "geos": ["EU-RU"]},
{"pool": "pool-2", "geos": ["EU"]},
{"pool": "other"},
],
},
},
),
)
wanted.add_record(
Record.new(
wanted,
"geo-cname-defaults",
{
"ttl": 300,
"type": "CNAME",
"value": "en.unit.tests.",
"dynamic": {
"pools": {
"pool-1": {
"values": [
{"value": "eu.unit.tests."},
],
},
},
"rules": [
{"pool": "pool-1", "geos": ["EU"]},
],
},
},
),
)
plan = provider.plan(wanted)
self.assertTrue(plan.exists)
self.assertEqual(4, len(plan.changes))
self.assertEqual(4, provider.apply(plan))
provider._client._request.assert_has_calls(
[
call(
"POST",
"http://api/zones/unit.tests/geo-simple.unit.tests./A",
data={
"ttl": 300,
"filters": [
{"type": "geodns"},
{"type": "first_n", "limit": 1},
{
"type": "default",
"limit": 1,
"strict": False,
},
],
"resource_records": [
{
"content": ["1.1.1.1"],
"meta": {"countries": ["RU"]},
},
{
"content": ["1.1.1.2"],
"meta": {"countries": ["RU"]},
},
{
"content": ["2.2.2.1"],
"meta": {"continents": ["EU"]},
},
{
"content": ["3.3.3.3"],
"meta": {"default": True},
},
],
},
),
call(
"POST",
"http://api/zones/unit.tests/geo-defaults.unit.tests./A",
data={
"ttl": 300,
"filters": [
{"type": "geodns"},
{"type": "first_n", "limit": 1},
{
"type": "default",
"limit": 1,
"strict": False,
},
],
"resource_records": [
{
"content": ["2.2.2.1"],
"meta": {"continents": ["EU"]},
},
{
"content": ["3.2.3.4"],
},
],
},
),
call(
"POST",
"http://api/zones/unit.tests/geo-cname-simple.unit.tests./CNAME",
data={
"ttl": 300,
"filters": [
{"type": "geodns"},
{"type": "first_n", "limit": 1},
{
"type": "default",
"limit": 1,
"strict": False,
},
],
"resource_records": [
{
"content": ["ru-1.unit.tests."],
"meta": {"countries": ["RU"]},
},
{
"content": ["ru-2.unit.tests."],
"meta": {"countries": ["RU"]},
},
{
"content": ["eu.unit.tests."],
"meta": {"continents": ["EU"]},
},
{
"content": ["en.unit.tests."],
"meta": {"default": True},
},
],
},
),
call(
"POST",
"http://api/zones/unit.tests/geo-cname-defaults.unit.tests./CNAME",
data={
"ttl": 300,
"filters": [
{"type": "geodns"},
{"type": "first_n", "limit": 1},
{
"type": "default",
"limit": 1,
"strict": False,
},
],
"resource_records": [
{
"content": ["eu.unit.tests."],
"meta": {"continents": ["EU"]},
},
{
"content": ["en.unit.tests."],
},
],
},
),
]
)