1
0
mirror of https://github.com/github/octodns.git synced 2024-05-11 05:55:00 +00:00

Remove debugging prints, test dyn dynamic, fix problems found by tests

This commit is contained in:
Ross McFarland
2018-12-17 13:00:00 -08:00
parent b7eaa8b580
commit 942edd66c0
4 changed files with 516 additions and 115 deletions

View File

@@ -22,9 +22,6 @@ from ..record.geo import GeoCodes
from .base import BaseProvider
from pprint import pprint
###############################################################################
#
# The following monkey patching is to work around functionality that is lacking
@@ -409,7 +406,6 @@ class DynProvider(BaseProvider):
tds[fqdn][_type] = td
self._traffic_directors = dict(tds)
pprint(self._traffic_directors)
return self._traffic_directors
def _populate_geo_traffic_director(self, zone, fqdn, _type, td, lenient):
@@ -452,14 +448,14 @@ class DynProvider(BaseProvider):
return record
def _value_for_single(self, _type, record):
def _value_for_address(self, _type, record):
return {
'value': record.address,
'weight': record.weight,
}
_value_for_A = _value_for_single
_value_for_AAAA = _value_for_single
_value_for_A = _value_for_address
_value_for_AAAA = _value_for_address
def _value_for_CNAME(self, _type, record):
return {
@@ -467,34 +463,9 @@ class DynProvider(BaseProvider):
'weight': record.weight,
}
def _populate_dynamic_traffic_director(self, zone, fqdn, _type, td,
lenient):
# critical to call rulesets once, each call loads them :-(
rulesets = td.rulesets
# We'll go ahead and grab pools too, using all will include unref'd
# pools
response_pools = td.all_response_pools
pprint({
'rulesets': rulesets,
'response_pools': response_pools,
})
# We start out with something that will always change show
# change in case this is a busted TD. This will prevent us from
# creating a duplicate td. We'll overwrite this with real data
# provide we have it
def _populate_dynamic_pools(self, _type, rulesets, response_pools):
default = {}
pools = {}
rules = []
values = []
data = {
'dynamic': {
'pools': pools,
'rules': rules,
},
'type': _type,
'ttl': td.ttl,
}
data_for = getattr(self, '_data_for_{}'.format(_type))
value_for = getattr(self, '_value_for_{}'.format(_type))
@@ -505,27 +476,22 @@ class DynProvider(BaseProvider):
for response_pool in response_pools:
# We have to refresh the response pool to have access to its
# rs_chains and thus records, yeah... :-(
# TODO: look at rulesets first b/c they won't need a refresh...
response_pool.refresh()
pprint({
'reponse_pool': response_pool,
'response_pool.label': response_pool.label,
'rs_chains': response_pool.rs_chains,
})
try:
record_set = response_pool.rs_chains[0] \
.record_sets[0]
except IndexError:
# problems indicate a malformed ruleset, ignore it
self.log.warn('_populate_dynamic_traffic_director: '
'malformed response_pool "{}" ignoring',
'malformed response_pool "%s" ignoring',
response_pool.label)
continue
label = response_pool.label
pprint(label)
if label == 'default':
data.update(data_for(_type, record_set.records))
default = data_for(_type, record_set.records)
else:
if label not in pools:
# First time we've seen it get its data
@@ -536,29 +502,28 @@ class DynProvider(BaseProvider):
for r in record_set.records]
}
for ruleset in rulesets:
pprint({
'ruleset': ruleset,
'ruleset.label': ruleset.label,
'ruleset.criteria_type': ruleset.criteria_type,
'ruleset.criterial': ruleset.criteria,
})
return default, pools
def _populate_dynamic_rules(self, rulesets, pools):
rules = []
for ruleset in rulesets:
if ruleset.label.startswith('default:'):
continue
num_pools = len(ruleset.response_pools)
if num_pools > 1:
if num_pools > 0:
pool = ruleset.response_pools[0].label
# TODO: verify pool exists
if num_pools > 1:
# We have a fallback, record it in the approrpriate pool
fallback = ruleset.response_pools[1].label
# TODO: verify fallback exists
if fallback != 'default':
pools[pool]['fallback'] = fallback
elif num_pools > 0:
pool = ruleset.response_pools[0].label
else:
self.log.warn('_populate_dynamic_traffic_director: '
'ruleset "{}" has no response_pools',
'ruleset "%s" has no response_pools',
ruleset.label)
continue
@@ -580,20 +545,49 @@ class DynProvider(BaseProvider):
geos.append(GeoCodes.province_to_code(code.upper()))
for code in geo['region']:
geos.append(self.REGION_CODES_LOOKUP[int(code)])
geos.sort()
rule['geos'] = geos
elif criteria_type == 'always':
pass
else:
self.log.warn('_populate_dynamic_traffic_director: '
'unsupported criteria_type "{}", ignoring',
'unsupported criteria_type "%s", ignoring',
criteria_type)
continue
rules.append(rule)
pprint({
'record data': data
})
return rules
def _populate_dynamic_traffic_director(self, zone, fqdn, _type, td,
lenient):
# critical to call rulesets once, each call loads them :-(
rulesets = td.rulesets
# We'll go ahead and grab pools too, using all will include unref'd
# pools
response_pools = td.all_response_pools
# Populate pools
default, pools = self._populate_dynamic_pools(_type, rulesets,
response_pools)
# Populate rules
rules = self._populate_dynamic_rules(rulesets, pools)
# We start out with something that will always change show
# change in case this is a busted TD. This will prevent us from
# creating a duplicate td. We'll overwrite this with real data
# provide we have it
data = {
'dynamic': {
'pools': pools,
'rules': rules,
},
'type': _type,
'ttl': td.ttl,
}
# Include default's information in data
data.update(default)
name = zone.hostname_from_fqdn(fqdn)
record = Record.new(zone, name, data, source=self, lenient=lenient)
@@ -851,17 +845,17 @@ class DynProvider(BaseProvider):
pool.create(td)
return pool
def _records_for_A(self, values, record_extras):
def _dynamic_records_for_A(self, values, record_extras):
return [DSFARecord(v['value'], weight=v.get('weight', 1),
**record_extras)
for v in values]
def _records_for_AAAA(self, values, record_extras):
def _dynamic_records_for_AAAA(self, values, record_extras):
return [DSFAAAARecord(v['value'], weight=v.get('weight', 1),
**record_extras)
for v in values]
def _records_for_CNAME(self, record, record_extras):
def _dynamic_records_for_CNAME(self, values, record_extras):
return [DSFCNAMERecord(v['value'], weight=v.get('weight', 1),
**record_extras)
for v in values]
@@ -875,12 +869,9 @@ class DynProvider(BaseProvider):
values.sort(key=weighted_keyer)
print('*** looking for {}'.format(label))
for pool in pools:
if pool.label != label:
print(' != {}'.format(pool.label))
continue
print(' == {}'.format(pool.label))
try:
records = pool.rs_chains[0].record_sets[0].records
except IndexError:
@@ -888,15 +879,12 @@ class DynProvider(BaseProvider):
continue
value_for = getattr(self, '_value_for_{}'.format(_type))
record_values = [value_for(_type, r) for r in records]
pprint(record_values)
if record_values == values:
print(' match {} == {}'.format(record_values, values))
# it's a match
return pool
print(' not match {} != {}'.format(record_values, values))
# we need to create the pool
records_for = getattr(self, '_records_for_{}'.format(_type))
records_for = getattr(self, '_dynamic_records_for_{}'.format(_type))
records = records_for(values, record_extras)
record_set = DSFRecordSet(_type, label,
serve_count=min(len(records), 2),
@@ -1066,7 +1054,6 @@ class DynProvider(BaseProvider):
del fqdn_tds[_type]
def _mod_dynamic_rulesets(self, td, change):
print('\n\n*****\n\n')
new = change.new
# TODO: make sure we can update TTLs
@@ -1105,9 +1092,6 @@ class DynProvider(BaseProvider):
pools[rpid] = get_response_pool(rpid, td)
# now that we have full objects for the complete set of existing pools,
# a list will be more useful
pprint({
'pools': pools
})
pools = pools.values()
# Rulesets
@@ -1160,9 +1144,6 @@ class DynProvider(BaseProvider):
# Make sure we have all the pools we're going to need
for _id, pool in sorted(new.dynamic.pools.items()):
pprint({
_id, pool,
})
values = [{
'weight': v.get('weight', 1),
'value': v['value'],
@@ -1178,18 +1159,12 @@ class DynProvider(BaseProvider):
criteria_type = 'always'
try:
geos = rule.data['geos']
pprint({
'geos': geos
})
criteria_type = 'geoip'
except KeyError:
geos = []
for geo in geos:
geo = GeoCodes.parse(geo)
pprint({
'geo': geo
})
if geo['province_code']:
criteria['geoip']['province'] \
.append(geo['province_code'].lower())
@@ -1200,11 +1175,6 @@ class DynProvider(BaseProvider):
criteria['geoip']['region'] \
.append(self.REGION_CODES[geo['continent_code']])
pprint({
'type': criteria_type,
'criteria': criteria
})
label = '{}:{}'.format(rule_num, uuid4().hex)
ruleset = DSFRuleset(label, criteria_type, [], criteria)
# Something you have to call create others the constructor does it
@@ -1312,7 +1282,6 @@ class DynProvider(BaseProvider):
self.log.debug('_apply_traffic_directors: zone=%s', desired.name)
unhandled_changes = []
for c in changes:
pprint(c)
# we only mess with changes that have geo info somewhere
if getattr(c.new, 'dynamic', False) or getattr(c.existing,
'dynamic', False):

View File

@@ -12,9 +12,6 @@ import re
from .geo import GeoCodes
from pprint import pprint
class Change(object):
def __init__(self, existing, new):
@@ -270,10 +267,7 @@ class _ValuesMixin(object):
try:
values = data['values']
except KeyError:
try:
values = [data['value']]
except KeyError:
values = []
self.values = sorted(self._value_type.process(values))
def changes(self, other, target):
@@ -367,10 +361,7 @@ class _ValueMixin(object):
def __init__(self, zone, name, data, source=None):
super(_ValueMixin, self).__init__(zone, name, data, source=source)
if 'value' in data:
self.value = self._value_type.process(data['value'])
else:
self.value = None
def changes(self, other, target):
if self.value != other.value:
@@ -394,8 +385,6 @@ class _DynamicPool(object):
def __init__(self, _id, data):
self._id = _id
pprint(['before', data])
values = [
{
'value': d['value'],
@@ -410,8 +399,6 @@ class _DynamicPool(object):
'values': values,
}
pprint(['after', self.data])
def _data(self):
return self.data
@@ -430,8 +417,6 @@ class _DynamicRule(object):
def __init__(self, i, data):
self.i = i
pprint(['before', data])
self.data = {}
try:
self.data['pool'] = data['pool']
@@ -442,17 +427,10 @@ class _DynamicRule(object):
except KeyError:
pass
pprint(['after', self.data])
def _data(self):
return self.data
def __eq__(self, other):
pprint([
self.data,
other.data,
self.data == other.data,
])
return self.data == other.data
def __ne__(self, other):

View File

@@ -1577,3 +1577,458 @@ class TestDSFMonitorMonkeyPatching(TestCase):
monitor = DummyDSFMonitor()
monitor.port = 8080
self.assertEquals(8080, monitor.port)
class DummyRecord(object):
def __init__(self, address, weight, ttl):
self.address = address
self.weight = weight
self.ttl = ttl
class DummyRecordSets(object):
def __init__(self, records):
self.records = records
class DummyRsChains(object):
def __init__(self, records):
self.record_sets = [DummyRecordSets(records)]
class DummyResponsePool(object):
def __init__(self, label, records=[]):
self.label = label
if records:
self.rs_chains = [DummyRsChains(records)]
else:
self.rs_chains = []
def refresh(self):
pass
class DummyRuleset(object):
def __init__(self, label, response_pools=[],
criteria_type='always', criteria={}):
self.label = label
self.response_pools = response_pools
self.criteria_type = criteria_type
self.criteria = criteria
class DummyTrafficDirector(object):
def __init__(self, rulesets=[], response_pools=[], ttl=42):
self.label = 'dynamic:dummy'
self.rulesets = rulesets
self.all_response_pools = response_pools
self.ttl = ttl
class TestDynProviderDynamic(TestCase):
def test_value_for_address(self):
provider = DynProvider('test', 'cust', 'user', 'pass')
class DummyRecord(object):
def __init__(self, address, weight):
self.address = address
self.weight = weight
record = DummyRecord('1.2.3.4', 32)
self.assertEquals({
'value': record.address,
'weight': record.weight,
}, provider._value_for_A('A', record))
record = DummyRecord('2601:644:500:e210:62f8:1dff:feb8:947a', 32)
self.assertEquals({
'value': record.address,
'weight': record.weight,
}, provider._value_for_AAAA('AAAA', record))
def test_value_for_CNAME(self):
provider = DynProvider('test', 'cust', 'user', 'pass')
class DummyRecord(object):
def __init__(self, cname, weight):
self.cname = cname
self.weight = weight
record = DummyRecord('foo.unit.tests.', 32)
self.assertEquals({
'value': record.cname,
'weight': record.weight,
}, provider._value_for_CNAME('CNAME', record))
def test_populate_dynamic_pools(self):
provider = DynProvider('test', 'cust', 'user', 'pass')
# Empty data, empty returns
default, pools = provider._populate_dynamic_pools('A', [], [])
self.assertEquals({}, default)
self.assertEquals({}, pools)
records_a = [DummyRecord('1.2.3.4', 32, 60)]
default_a = DummyResponsePool('default', records_a)
# Just a default A
response_pools = [default_a]
default, pools = provider._populate_dynamic_pools('A', [],
response_pools)
self.assertEquals({
'ttl': 60,
'type': 'A',
'values': ['1.2.3.4'],
}, default)
self.assertEquals({}, pools)
multi_a = [
DummyRecord('1.2.3.5', 42, 90),
DummyRecord('1.2.3.6', 43, 90),
DummyRecord('1.2.3.7', 44, 90),
]
example_a = DummyResponsePool('example', multi_a)
# Just a named pool
response_pools = [example_a]
default, pools = provider._populate_dynamic_pools('A', [],
response_pools)
self.assertEquals({}, default)
self.assertEquals({
'example': {
'values': [{
'value': '1.2.3.5',
'weight': 42,
}, {
'value': '1.2.3.6',
'weight': 43,
}, {
'value': '1.2.3.7',
'weight': 44,
}],
},
}, pools)
# Named pool that shows up twice
response_pools = [example_a, example_a]
default, pools = provider._populate_dynamic_pools('A', [],
response_pools)
self.assertEquals({}, default)
self.assertEquals({
'example': {
'values': [{
'value': '1.2.3.5',
'weight': 42,
}, {
'value': '1.2.3.6',
'weight': 43,
}, {
'value': '1.2.3.7',
'weight': 44,
}],
},
}, pools)
# Default & named
response_pools = [example_a, default_a, example_a]
default, pools = provider._populate_dynamic_pools('A', [],
response_pools)
self.assertEquals({
'ttl': 60,
'type': 'A',
'values': ['1.2.3.4'],
}, default)
self.assertEquals({
'example': {
'values': [{
'value': '1.2.3.5',
'weight': 42,
}, {
'value': '1.2.3.6',
'weight': 43,
}, {
'value': '1.2.3.7',
'weight': 44,
}],
},
}, pools)
# empty rs_chains doesn't cause an example, just ignores
empty_a = DummyResponsePool('empty')
response_pools = [empty_a]
default, pools = provider._populate_dynamic_pools('A', [],
response_pools)
self.assertEquals({}, default)
self.assertEquals({}, pools)
def test_populate_dynamic_rules(self):
provider = DynProvider('test', 'cust', 'user', 'pass')
# Empty
rulesets = []
pools = {}
rules = provider._populate_dynamic_rules(rulesets, pools)
self.assertEquals([], rules)
# default: is ignored
rulesets = [DummyRuleset('default:')]
pools = {}
rules = provider._populate_dynamic_rules(rulesets, pools)
self.assertEquals([], rules)
# No ResponsePools in RuleSet, ignored
rulesets = [DummyRuleset('0:abcdefg')]
pools = {}
rules = provider._populate_dynamic_rules(rulesets, pools)
self.assertEquals([], rules)
# ResponsePool, no fallback
rulesets = [DummyRuleset('0:abcdefg', [
DummyResponsePool('some-pool')
])]
pools = {}
rules = provider._populate_dynamic_rules(rulesets, pools)
self.assertEquals([{
'pool': 'some-pool',
}], rules)
# ResponsePool, with dfault fallback (ignored)
rulesets = [DummyRuleset('0:abcdefg', [
DummyResponsePool('some-pool'),
DummyResponsePool('default'),
])]
pools = {}
rules = provider._populate_dynamic_rules(rulesets, pools)
self.assertEquals([{
'pool': 'some-pool',
}], rules)
# ResponsePool, with fallback
rulesets = [DummyRuleset('0:abcdefg', [
DummyResponsePool('some-pool'),
DummyResponsePool('some-fallback'),
])]
pools = {
'some-pool': {},
}
rules = provider._populate_dynamic_rules(rulesets, pools)
self.assertEquals([{
'pool': 'some-pool',
}], rules)
# fallback has been installed
self.assertEquals({
'some-pool': {
'fallback': 'some-fallback',
}
}, pools)
# Unsupported criteria_type (ignored)
rulesets = [DummyRuleset('0:abcdefg', [
DummyResponsePool('some-pool')
], 'unsupported')]
pools = {}
rules = provider._populate_dynamic_rules(rulesets, pools)
self.assertEquals([], rules)
# Geo Continent/Region
response_pools = [DummyResponsePool('some-pool')]
criteria = {
'geoip': {
'country': ['US'],
'province': ['or'],
'region': [14],
},
}
ruleset = DummyRuleset('0:abcdefg', response_pools,
'geoip', criteria)
rulesets = [ruleset]
pools = {}
rules = provider._populate_dynamic_rules(rulesets, pools)
self.assertEquals([{
'geos': ['AF', 'NA-US', 'NA-US-OR'],
'pool': 'some-pool',
}], rules)
def test_populate_dynamic_traffic_director(self):
provider = DynProvider('test', 'cust', 'user', 'pass')
fqdn = 'dynamic.unit.tests.'
multi_a = [
DummyRecord('1.2.3.5', 1, 90),
DummyRecord('1.2.3.6', 1, 90),
DummyRecord('1.2.3.7', 1, 90),
]
default_response_pool = DummyResponsePool('default', multi_a)
pool1_response_pool = DummyResponsePool('pool1', multi_a)
rulesets = [
DummyRuleset('default', [default_response_pool]),
DummyRuleset('0:abcdef', [pool1_response_pool], 'geoip', {
'geoip': {
'country': ['US'],
'province': ['or'],
'region': [14],
},
}),
]
td = DummyTrafficDirector(rulesets, [default_response_pool,
pool1_response_pool])
zone = Zone('unit.tests.', [])
record = provider._populate_dynamic_traffic_director(zone, fqdn, 'A',
td, True)
self.assertTrue(record)
self.assertEquals('A', record._type)
self.assertEquals(90, record.ttl)
self.assertEquals([
'1.2.3.5',
'1.2.3.6',
'1.2.3.7',
], record.values)
self.assertTrue('pool1' in record.dynamic.pools)
self.assertEquals({
'fallback': None,
'values': [{
'value': '1.2.3.5',
'weight': 1,
}, {
'value': '1.2.3.6',
'weight': 1,
}, {
'value': '1.2.3.7',
'weight': 1,
}]
}, record.dynamic.pools['pool1'].data)
self.assertEquals(2, len(record.dynamic.rules))
self.assertEquals({
'pool': 'default',
}, record.dynamic.rules[0].data)
self.assertEquals({
'pool': 'pool1',
'geos': ['AF', 'NA-US', 'NA-US-OR'],
}, record.dynamic.rules[1].data)
# Hack into the provider and create a fake list of traffic directors
provider._traffic_directors = {
'dynamic.unit.tests.': {
'A': td,
}
}
zone = Zone('unit.tests.', [])
records = provider._populate_traffic_directors(zone, lenient=True)
self.assertEquals(1, len(records))
def test_dynamic_records_for_A(self):
provider = DynProvider('test', 'cust', 'user', 'pass')
# Empty
records = provider._dynamic_records_for_A([], {})
self.assertEquals([], records)
# Basic
values = [{
'value': '1.2.3.4',
}, {
'value': '1.2.3.5',
'weight': 42,
}]
records = provider._dynamic_records_for_A(values, {})
self.assertEquals(2, len(records))
record = records[0]
self.assertEquals('1.2.3.4', record.address)
self.assertEquals(1, record.weight)
record = records[1]
self.assertEquals('1.2.3.5', record.address)
self.assertEquals(42, record.weight)
# With extras
records = provider._dynamic_records_for_A(values, {
'automation': 'manual',
'eligible': True,
})
self.assertEquals(2, len(records))
record = records[0]
self.assertEquals('1.2.3.4', record.address)
self.assertEquals(1, record.weight)
self.assertEquals('manual', record._automation)
self.assertTrue(record.eligible)
def test_dynamic_records_for_AAAA(self):
provider = DynProvider('test', 'cust', 'user', 'pass')
# Empty
records = provider._dynamic_records_for_AAAA([], {})
self.assertEquals([], records)
# Basic
values = [{
'value': '2601:644:500:e210:62f8:1dff:feb8:947a',
}, {
'value': '2601:644:500:e210:62f8:1dff:feb8:947b',
'weight': 42,
}]
records = provider._dynamic_records_for_AAAA(values, {})
self.assertEquals(2, len(records))
record = records[0]
self.assertEquals('2601:644:500:e210:62f8:1dff:feb8:947a',
record.address)
self.assertEquals(1, record.weight)
record = records[1]
self.assertEquals('2601:644:500:e210:62f8:1dff:feb8:947b',
record.address)
self.assertEquals(42, record.weight)
# With extras
records = provider._dynamic_records_for_AAAA(values, {
'automation': 'manual',
'eligible': True,
})
self.assertEquals(2, len(records))
record = records[0]
self.assertEquals('2601:644:500:e210:62f8:1dff:feb8:947a',
record.address)
self.assertEquals(1, record.weight)
self.assertEquals('manual', record._automation)
self.assertTrue(record.eligible)
def test_dynamic_records_for_CNAME(self):
provider = DynProvider('test', 'cust', 'user', 'pass')
# Empty
records = provider._dynamic_records_for_CNAME([], {})
self.assertEquals([], records)
# Basic
values = [{
'value': 'target-1.unit.tests.',
}, {
'value': 'target-2.unit.tests.',
'weight': 42,
}]
records = provider._dynamic_records_for_CNAME(values, {})
self.assertEquals(2, len(records))
record = records[0]
self.assertEquals('target-1.unit.tests.', record.cname)
self.assertEquals(1, record.weight)
record = records[1]
self.assertEquals('target-2.unit.tests.', record.cname)
self.assertEquals(42, record.weight)
# With extras
records = provider._dynamic_records_for_CNAME(values, {
'automation': 'manual',
'eligible': True,
})
self.assertEquals(2, len(records))
record = records[0]
self.assertEquals('target-1.unit.tests.', record.cname)
self.assertEquals(1, record.weight)
self.assertEquals('manual', record._automation)
self.assertTrue(record.eligible)

View File

@@ -170,7 +170,6 @@ class TestEtcHostsProvider(TestCase):
with open(hosts_file) as fh:
data = fh.read()
print(data)
self.assertTrue('# loop.unit.tests -> start.unit.tests '
'**loop**' in data)
self.assertTrue('# middle.unit.tests -> loop.unit.tests '