mirror/github-octodns
1
0
Fork 0
mirror of https://github.com/github/octodns.git synced 2024-05-11 05:55:00 +00:00
Code Issues Releases Activity

Handle multiples sources on aliased zones

Browse Source
This commit is contained in:
Jonathan Leroy
2020-10-31 01:09:37 +01:00
parent 04a2d7a585
commit 3acea0d89d
3 changed files with 82 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
octodns/manager.py
View File

@@ -121,23 +121,6 @@ class Manager(object):
raise ManagerException('Incorrect provider config for {}' raise ManagerException('Incorrect provider config for {}'
.format(provider_name)) .format(provider_name))
for zone_name, zone_config in self.config['zones'].copy().items():
if 'alias' in zone_config:
source_zone = zone_config['alias']
# Check that the source zone is defined.
if source_zone not in self.config['zones']:
self.log.exception('Invalid alias zone')
raise ManagerException('Invalid alias zone {}: '
'source zone {} does not exist'
.format(zone_name, source_zone))
self.config['zones'][zone_name] = \
self.config['zones'][source_zone]
self.config['zones'][zone_name]['is_alias'] = True
self.config['zones'][zone_name]['file'] = source_zone
else:
self.config['zones'][zone_name]['is_alias'] = False
self.config['zones'][zone_name]['file'] = zone_name
zone_tree = {} zone_tree = {}
# sort by reversed strings so that parent zones always come first # sort by reversed strings so that parent zones always come first
for name in sorted(self.config['zones'].keys(), key=lambda s: s[::-1]): for name in sorted(self.config['zones'].keys(), key=lambda s: s[::-1]):
@@ -239,23 +222,32 @@ class Manager(object):
self.log.debug('configured_sub_zones: subs=%s', sub_zone_names) self.log.debug('configured_sub_zones: subs=%s', sub_zone_names)
return set(sub_zone_names) return set(sub_zone_names)
def _populate_and_plan(self, zone_name, file, is_alias, sources, targets, def _populate_and_plan(self, zone_name, sources, targets, desired=None,
lenient=False): lenient=False):
self.log.debug('sync: populating, zone=%s, file=%s, is_alias=%s, ' self.log.debug('sync: populating, zone=%s, lenient=%s',
'lenient=%s', zone_name, file, is_alias, lenient) zone_name, lenient)
zone = Zone(zone_name, zone = Zone(zone_name,
sub_zones=self.configured_sub_zones(zone_name), file=file, sub_zones=self.configured_sub_zones(zone_name))
is_alias=is_alias)
for source in sources: if not desired:
try: for source in sources:
source.populate(zone, lenient=lenient) try:
except TypeError as e: source.populate(zone, lenient=lenient)
if "keyword argument 'lenient'" not in text_type(e): except TypeError as e:
raise if "keyword argument 'lenient'" not in text_type(e):
self.log.warn(': provider %s does not accept lenient param', raise
source.__class__.__name__) self.log.warn(': provider %s does not accept lenient '
source.populate(zone) 'param', source.__class__.__name__)
source.populate(zone)
else:
for _, records in desired._records.items():
for record in records:
d = record.data
d['type'] = record._type
r = Record.new(zone, record.name, d, source=record.source)
zone.add_record(r, lenient=lenient)
self.log.debug('sync: planning, zone=%s', zone_name) self.log.debug('sync: planning, zone=%s', zone_name)
plans = [] plans = []
@@ -284,11 +276,22 @@ class Manager(object):
if eligible_zones: if eligible_zones:
zones = [z for z in zones if z[0] in eligible_zones] zones = [z for z in zones if z[0] in eligible_zones]
aliased_zones = {}
futures = [] futures = []
for zone_name, config in zones: for zone_name, config in zones:
self.log.info('sync: zone=%s', zone_name) self.log.info('sync: zone=%s', zone_name)
file = config.get('file') if 'alias' in config:
is_alias = config.get('is_alias') source_zone = config['alias']
# Check that the source zone is defined.
if source_zone not in self.config['zones']:
self.log.exception('Invalid alias zone')
raise ManagerException('Invalid alias zone {}: '
'source zone {} does not exist'
.format(zone_name, source_zone))
aliased_zones[zone_name] = source_zone
continue
lenient = config.get('lenient', False) lenient = config.get('lenient', False)
try: try:
sources = config['sources'] sources = config['sources']
@@ -345,14 +348,32 @@ class Manager(object):
.format(zone_name, target)) .format(zone_name, target))
futures.append(self._executor.submit(self._populate_and_plan, futures.append(self._executor.submit(self._populate_and_plan,
zone_name, file, is_alias, zone_name, sources,
sources, targets, targets, lenient=lenient))
lenient=lenient))
# Wait on all results and unpack/flatten them in to a list of target & # Wait on all results and unpack/flatten them in to a list of target &
# plan pairs. # plan pairs.
plans = [p for f in futures for p in f.result()] plans = [p for f in futures for p in f.result()]
# Populate aliases zones.
futures = []
for zone_name, zone_source in aliased_zones.items():
plan = [p for t, p in plans if p.desired.name == zone_source]
if not plan:
continue
source_config = self.config['zones'][zone_source]
futures.append(self._executor.submit(
self._populate_and_plan,
zone_name,
[self.providers[s] for s in source_config['sources']],
[self.providers[t] for t in source_config['targets']],
desired=plan[0].desired,
lenient=lenient
))
plans += [p for f in futures for p in f.result()]
# Best effort sort plans children first so that we create/update # Best effort sort plans children first so that we create/update
# children zones before parents which should allow us to more safely # children zones before parents which should allow us to more safely
# extract things into sub-zones. Combining a child back into a parent # extract things into sub-zones. Combining a child back into a parent
@@ -440,32 +461,30 @@ class Manager(object):
def validate_configs(self): def validate_configs(self):
for zone_name, config in self.config['zones'].items(): for zone_name, config in self.config['zones'].items():
file = config.get('file', False) zone = Zone(zone_name, self.configured_sub_zones(zone_name))
is_alias = config.get('is_alias', False)
zone = Zone(zone_name, self.configured_sub_zones(zone_name),
file, is_alias)
try: if not config.get('alias'):
sources = config['sources'] try:
except KeyError: sources = config['sources']
raise ManagerException('Zone {} is missing sources' except KeyError:
.format(zone_name)) raise ManagerException('Zone {} is missing sources'
.format(zone_name))
try:
# rather than using a list comprehension, we break this
# loop out so that the `except` block below can reference
# the `source`
collected = []
for source in sources:
collected.append(self.providers[source])
sources = collected
except KeyError:
raise ManagerException('Zone {}, unknown source: {}'
.format(zone_name, source))
try:
# rather than using a list comprehension, we break this loop
# out so that the `except` block below can reference the
# `source`
collected = []
for source in sources: for source in sources:
collected.append(self.providers[source]) if isinstance(source, YamlProvider):
sources = collected source.populate(zone)
except KeyError:
raise ManagerException('Zone {}, unknown source: {}'
.format(zone_name, source))
for source in sources:
if isinstance(source, YamlProvider):
source.populate(zone)
def get_zone(self, zone_name): def get_zone(self, zone_name):
if not zone_name[-1] == '.': if not zone_name[-1] == '.':
@@ -474,10 +493,6 @@ class Manager(object):
for name, config in self.config['zones'].items(): for name, config in self.config['zones'].items():
if name == zone_name: if name == zone_name:
file = config.get('file', False) return Zone(name, self.configured_sub_zones(name))
is_alias = config.get('is_alias', False)
return Zone(name, self.configured_sub_zones(name),
file, is_alias)
raise ManagerException('Unknown zone name {}'.format(zone_name)) raise ManagerException('Unknown zone name {}'.format(zone_name))

5
octodns/provider/yaml.py
View File

@@ -139,8 +139,7 @@ class YamlProvider(BaseProvider):
filename) filename)
def populate(self, zone, target=False, lenient=False): def populate(self, zone, target=False, lenient=False):
self.log.debug('populate: name=%s, file=%s, is_alias:%s, target=%s, ' self.log.debug('populate: name=%s, target=%s, lenient=%s', zone.name,
'lenient=%s', zone.name, zone.file, zone.is_alias,
target, lenient) target, lenient)
if target: if target:
@@ -149,7 +148,7 @@ class YamlProvider(BaseProvider):
return False return False
before = len(zone.records) before = len(zone.records)
filename = join(self.directory, '{}yaml'.format(zone.file)) filename = join(self.directory, '{}yaml'.format(zone.name))
self._populate_from_file(filename, zone, lenient) self._populate_from_file(filename, zone, lenient)
self.log.info('populate: found %s records, exists=False', self.log.info('populate: found %s records, exists=False',

4
octodns/zone.py
View File

@@ -35,15 +35,13 @@ def _is_eligible(record):
class Zone(object): class Zone(object):
log = getLogger('Zone') log = getLogger('Zone')
def __init__(self, name, sub_zones, file=None, is_alias=False): def __init__(self, name, sub_zones):
if not name[-1] == '.': if not name[-1] == '.':
raise Exception('Invalid zone name {}, missing ending dot' raise Exception('Invalid zone name {}, missing ending dot'
.format(name)) .format(name))
# Force everything to lowercase just to be safe # Force everything to lowercase just to be safe
self.name = text_type(name).lower() if name else name self.name = text_type(name).lower() if name else name
self.sub_zones = sub_zones self.sub_zones = sub_zones
self.file = text_type(file if file else name).lower()
self.is_alias = is_alias
# We're grouping by node, it allows us to efficiently search for # We're grouping by node, it allows us to efficiently search for
# duplicates and detect when CNAMEs co-exist with other records # duplicates and detect when CNAMEs co-exist with other records
self._records = defaultdict(set) self._records = defaultdict(set)