mirror/github-octodns
1
0
Fork 0
mirror of https://github.com/github/octodns.git synced 2024-05-11 05:55:00 +00:00
Code Issues Releases Activity

Implement simplistic planning concurrency, by zone

Browse Source
This commit is contained in:
Ross McFarland
2017-03-27 14:39:50 -07:00
parent 67e8139580
commit 377bda4f9c
3 changed files with 34 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
octodns/manager.py
View File

@@ -6,6 +6,7 @@ from __future__ import absolute_import, division, print_function, \
unicode_literals
from StringIO import StringIO
from concurrent.futures import ThreadPoolExecutor
from importlib import import_module
from os import environ
import logging
@@ -46,6 +47,10 @@ class Manager(object):
with open(config_file, 'r') as fh:
self.config = safe_load(fh, enforce_order=False)
manager_config = self.config.get('manager', {})
max_workers = manager_config.get('max_workers', 4)
self._executor = ThreadPoolExecutor(max_workers=max_workers)
self.log.debug('__init__: configuring providers')
self.providers = {}
for provider_name, provider_config in self.config['providers'].items():
@@ -135,6 +140,24 @@ class Manager(object):
self.log.debug('configured_sub_zones: subs=%s', sub_zone_names)
return set(sub_zone_names)
def _populate_and_plan(self, zone_name, sources, targets):
self.log.debug('sync: populating, zone=%s', zone_name)
zone = Zone(zone_name,
sub_zones=self.configured_sub_zones(zone_name))
for source in sources:
source.populate(zone)
self.log.debug('sync: planning, zone=%s', zone_name)
plans = []
for target in targets:
plan = target.plan(zone)
if plan:
plans.append((target, plan))
return plans
def sync(self, eligible_zones=[], eligible_targets=[], dry_run=True,
force=False):
self.log.info('sync: eligible_zones=%s, eligible_targets=%s, '
@@ -145,7 +168,7 @@ class Manager(object):
if eligible_zones:
zones = filter(lambda d: d[0] in eligible_zones, zones)
plans = []
futures = []
for zone_name, config in zones:
self.log.info('sync: zone=%s', zone_name)
try:
@@ -181,17 +204,12 @@ class Manager(object):
raise Exception('Zone {}, unknown target: {}'.format(zone_name,
target))
self.log.debug('sync: populating')
zone = Zone(zone_name,
sub_zones=self.configured_sub_zones(zone_name))
for source in sources:
source.populate(zone)
futures.append(self._executor.submit(self._populate_and_plan,
zone_name, sources, targets))
self.log.debug('sync: planning')
for target in targets:
plan = target.plan(zone)
if plan:
plans.append((target, plan))
# Wait on all results and unpack/flatten them in to a list of target &
# plan pairs.
plans = [p for f in futures for p in f.result()]
hr = '*************************************************************' \
'*******************\n'