diff --git a/.gitignore b/.gitignore index c45a684..64ce76f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.pyc .coverage .env +/config/ coverage.xml dist/ env/ @@ -9,5 +10,3 @@ nosetests.xml octodns.egg-info/ output/ tmp/ -build/ -config/ diff --git a/octodns/manager.py b/octodns/manager.py index 36a3592..d4debf6 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -5,13 +5,13 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -from StringIO import StringIO from concurrent.futures import ThreadPoolExecutor from importlib import import_module from os import environ import logging -from .provider.base import BaseProvider, Plan +from .provider.base import BaseProvider +from .provider.plan import Plan from .provider.yaml import YamlProvider from .record import Record from .yaml import safe_load @@ -95,23 +95,8 @@ class Manager(object): self.log.exception('Invalid provider class') raise Exception('Provider {} is missing class' .format(provider_name)) - _class = self._get_provider_class(_class) - # Build up the arguments we need to pass to the provider - kwargs = {} - for k, v in provider_config.items(): - try: - if v.startswith('env/'): - try: - env_var = v[4:] - v = environ[env_var] - except KeyError: - self.log.exception('Invalid provider config') - raise Exception('Incorrect provider config, ' - 'missing env var {}' - .format(env_var)) - except AttributeError: - pass - kwargs[k] = v + _class = self._get_named_class('provider', _class) + kwargs = self._build_kwargs(provider_config) try: self.providers[provider_name] = _class(provider_name, **kwargs) except TypeError: @@ -139,20 +124,64 @@ class Manager(object): where = where[piece] self.zone_tree = zone_tree - def _get_provider_class(self, _class): + self.plan_outputs = {} + plan_outputs = manager_config.get('plan_outputs', { + 'logger': { + 'class': 'octodns.provider.plan.PlanLogger', + 'level': 'info' + } + }) + for plan_output_name, plan_output_config in plan_outputs.items(): + try: + _class = plan_output_config.pop('class') + except KeyError: + self.log.exception('Invalid plan_output class') + raise Exception('plan_output {} is missing class' + .format(plan_output_name)) + _class = self._get_named_class('plan_output', _class) + kwargs = self._build_kwargs(plan_output_config) + try: + self.plan_outputs[plan_output_name] = \ + _class(plan_output_name, **kwargs) + except TypeError: + self.log.exception('Invalid plan_output config') + raise Exception('Incorrect plan_output config for {}' + .format(plan_output_name)) + + def _get_named_class(self, _type, _class): try: module_name, class_name = _class.rsplit('.', 1) module = import_module(module_name) except (ImportError, ValueError): - self.log.exception('_get_provider_class: Unable to import ' + self.log.exception('_get_{}_class: Unable to import ' 'module %s', _class) - raise Exception('Unknown provider class: {}'.format(_class)) + raise Exception('Unknown {} class: {}'.format(_type, _class)) try: return getattr(module, class_name) except AttributeError: - self.log.exception('_get_provider_class: Unable to get class %s ' + self.log.exception('_get_{}_class: Unable to get class %s ' 'from module %s', class_name, module) - raise Exception('Unknown provider class: {}'.format(_class)) + raise Exception('Unknown {} class: {}'.format(_type, _class)) + + def _build_kwargs(self, source): + # Build up the arguments we need to pass to the provider + kwargs = {} + for k, v in source.items(): + try: + if v.startswith('env/'): + try: + env_var = v[4:] + v = environ[env_var] + except KeyError: + self.log.exception('Invalid provider config') + raise Exception('Incorrect provider config, ' + 'missing env var {}' + .format(env_var)) + except AttributeError: + pass + kwargs[k] = v + + return kwargs def configured_sub_zones(self, zone_name): # Reversed pieces of the zone name @@ -259,39 +288,8 @@ class Manager(object): # plan pairs. plans = [p for f in futures for p in f.result()] - hr = '*************************************************************' \ - '*******************\n' - buf = StringIO() - buf.write('\n') - if plans: - current_zone = None - for target, plan in plans: - if plan.desired.name != current_zone: - current_zone = plan.desired.name - buf.write(hr) - buf.write('* ') - buf.write(current_zone) - buf.write('\n') - buf.write(hr) - - buf.write('* ') - buf.write(target.id) - buf.write(' (') - buf.write(target) - buf.write(')\n* ') - for change in plan.changes: - buf.write(change.__repr__(leader='* ')) - buf.write('\n* ') - - buf.write('Summary: ') - buf.write(plan) - buf.write('\n') - else: - buf.write(hr) - buf.write('No changes were planned\n') - buf.write(hr) - buf.write('\n') - self.log.info(buf.getvalue()) + for output in self.plan_outputs.values(): + output.run(plans=plans, log=self.log) if not force: self.log.debug('sync: checking safety') diff --git a/octodns/provider/base.py b/octodns/provider/base.py index f6ff1b7..2d4680f 100644 --- a/octodns/provider/base.py +++ b/octodns/provider/base.py @@ -7,78 +7,7 @@ from __future__ import absolute_import, division, print_function, \ from ..source.base import BaseSource from ..zone import Zone -from logging import getLogger - - -class UnsafePlan(Exception): - pass - - -class Plan(object): - log = getLogger('Plan') - - MAX_SAFE_UPDATE_PCENT = .3 - MAX_SAFE_DELETE_PCENT = .3 - MIN_EXISTING_RECORDS = 10 - - def __init__(self, existing, desired, changes, - update_pcent_threshold=MAX_SAFE_UPDATE_PCENT, - delete_pcent_threshold=MAX_SAFE_DELETE_PCENT): - self.existing = existing - self.desired = desired - self.changes = changes - self.update_pcent_threshold = update_pcent_threshold - self.delete_pcent_threshold = delete_pcent_threshold - - change_counts = { - 'Create': 0, - 'Delete': 0, - 'Update': 0 - } - for change in changes: - change_counts[change.__class__.__name__] += 1 - self.change_counts = change_counts - - try: - existing_n = len(self.existing.records) - except AttributeError: - existing_n = 0 - - self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d' - 'Existing=%d', - self.change_counts['Create'], - self.change_counts['Update'], - self.change_counts['Delete'], existing_n) - - def raise_if_unsafe(self): - # TODO: what is safe really? - if self.existing and \ - len(self.existing.records) >= self.MIN_EXISTING_RECORDS: - - existing_record_count = len(self.existing.records) - update_pcent = self.change_counts['Update'] / existing_record_count - delete_pcent = self.change_counts['Delete'] / existing_record_count - - if update_pcent > self.update_pcent_threshold: - raise UnsafePlan('Too many updates, {} is over {} percent' - '({}/{})'.format( - update_pcent, - self.MAX_SAFE_UPDATE_PCENT * 100, - self.change_counts['Update'], - existing_record_count)) - if delete_pcent > self.delete_pcent_threshold: - raise UnsafePlan('Too many deletes, {} is over {} percent' - '({}/{})'.format( - delete_pcent, - self.MAX_SAFE_DELETE_PCENT * 100, - self.change_counts['Delete'], - existing_record_count)) - - def __repr__(self): - return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \ - .format(self.change_counts['Create'], self.change_counts['Update'], - self.change_counts['Delete'], - len(self.existing.records)) +from .plan import Plan class BaseProvider(BaseSource): diff --git a/octodns/provider/plan.py b/octodns/provider/plan.py new file mode 100644 index 0000000..3e86826 --- /dev/null +++ b/octodns/provider/plan.py @@ -0,0 +1,266 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from StringIO import StringIO +from logging import DEBUG, ERROR, INFO, WARN, getLogger +from sys import stdout + + +class UnsafePlan(Exception): + pass + + +class Plan(object): + log = getLogger('Plan') + + MAX_SAFE_UPDATE_PCENT = .3 + MAX_SAFE_DELETE_PCENT = .3 + MIN_EXISTING_RECORDS = 10 + + def __init__(self, existing, desired, changes, + update_pcent_threshold=MAX_SAFE_UPDATE_PCENT, + delete_pcent_threshold=MAX_SAFE_DELETE_PCENT): + self.existing = existing + self.desired = desired + self.changes = changes + self.update_pcent_threshold = update_pcent_threshold + self.delete_pcent_threshold = delete_pcent_threshold + + change_counts = { + 'Create': 0, + 'Delete': 0, + 'Update': 0 + } + for change in changes: + change_counts[change.__class__.__name__] += 1 + self.change_counts = change_counts + + try: + existing_n = len(self.existing.records) + except AttributeError: + existing_n = 0 + + self.log.debug('__init__: Creates=%d, Updates=%d, Deletes=%d' + 'Existing=%d', + self.change_counts['Create'], + self.change_counts['Update'], + self.change_counts['Delete'], existing_n) + + def raise_if_unsafe(self): + # TODO: what is safe really? + if self.existing and \ + len(self.existing.records) >= self.MIN_EXISTING_RECORDS: + + existing_record_count = len(self.existing.records) + update_pcent = self.change_counts['Update'] / existing_record_count + delete_pcent = self.change_counts['Delete'] / existing_record_count + + if update_pcent > self.update_pcent_threshold: + raise UnsafePlan('Too many updates, {} is over {} percent' + '({}/{})'.format( + update_pcent, + self.MAX_SAFE_UPDATE_PCENT * 100, + self.change_counts['Update'], + existing_record_count)) + if delete_pcent > self.delete_pcent_threshold: + raise UnsafePlan('Too many deletes, {} is over {} percent' + '({}/{})'.format( + delete_pcent, + self.MAX_SAFE_DELETE_PCENT * 100, + self.change_counts['Delete'], + existing_record_count)) + + def __repr__(self): + return 'Creates={}, Updates={}, Deletes={}, Existing Records={}' \ + .format(self.change_counts['Create'], self.change_counts['Update'], + self.change_counts['Delete'], + len(self.existing.records)) + + +class _PlanOutput(object): + + def __init__(self, name): + self.name = name + + +class PlanLogger(_PlanOutput): + + def __init__(self, name, level='info'): + super(PlanLogger, self).__init__(name) + try: + self.level = { + 'debug': DEBUG, + 'info': INFO, + 'warn': WARN, + 'warning': WARN, + 'error': ERROR + }[level.lower()] + except (AttributeError, KeyError): + raise Exception('Unsupported level: {}'.format(level)) + + def run(self, log, plans, *args, **kwargs): + hr = '*************************************************************' \ + '*******************\n' + buf = StringIO() + buf.write('\n') + if plans: + current_zone = None + for target, plan in plans: + if plan.desired.name != current_zone: + current_zone = plan.desired.name + buf.write(hr) + buf.write('* ') + buf.write(current_zone) + buf.write('\n') + buf.write(hr) + + buf.write('* ') + buf.write(target.id) + buf.write(' (') + buf.write(target) + buf.write(')\n* ') + for change in plan.changes: + buf.write(change.__repr__(leader='* ')) + buf.write('\n* ') + + buf.write('Summary: ') + buf.write(plan) + buf.write('\n') + else: + buf.write(hr) + buf.write('No changes were planned\n') + buf.write(hr) + buf.write('\n') + log.log(self.level, buf.getvalue()) + + +def _value_stringifier(record, sep): + try: + values = [str(v) for v in record.values] + except AttributeError: + values = [record.value] + for code, gv in sorted(getattr(record, 'geo', {}).items()): + vs = ', '.join([str(v) for v in gv.values]) + values.append('{}: {}'.format(code, vs)) + return sep.join(values) + + +class PlanMarkdown(_PlanOutput): + + def run(self, plans, fh=stdout, *args, **kwargs): + if plans: + current_zone = None + for target, plan in plans: + if plan.desired.name != current_zone: + current_zone = plan.desired.name + fh.write('## ') + fh.write(current_zone) + fh.write('\n\n') + + fh.write('### ') + fh.write(target.id) + fh.write('\n\n') + + fh.write('| Operation | Name | Type | TTL | Value | Source |\n' + '|--|--|--|--|--|--|\n') + + for change in plan.changes: + existing = change.existing + new = change.new + record = change.record + fh.write('| ') + fh.write(change.__class__.__name__) + fh.write(' | ') + fh.write(record.name) + fh.write(' | ') + fh.write(record._type) + fh.write(' | ') + # TTL + if existing: + fh.write(str(existing.ttl)) + fh.write(' | ') + fh.write(_value_stringifier(existing, '; ')) + fh.write(' | |\n') + if new: + fh.write('| | | | ') + + if new: + fh.write(str(new.ttl)) + fh.write(' | ') + fh.write(_value_stringifier(new, '; ')) + fh.write(' | ') + fh.write(new.source.id) + fh.write(' |\n') + + fh.write('\nSummary: ') + fh.write(str(plan)) + fh.write('\n\n') + else: + fh.write('## No changes were planned\n') + + +class PlanHtml(_PlanOutput): + + def run(self, plans, fh=stdout, *args, **kwargs): + if plans: + current_zone = None + for target, plan in plans: + if plan.desired.name != current_zone: + current_zone = plan.desired.name + fh.write('

') + fh.write(current_zone) + fh.write('

\n') + + fh.write('

') + fh.write(target.id) + fh.write('''

+ + + + + + + + + +''') + + for change in plan.changes: + existing = change.existing + new = change.new + record = change.record + fh.write(' \n \n \n \n') + # TTL + if existing: + fh.write(' \n \n \n \n') + if new: + fh.write(' \n \n') + + if new: + fh.write(' \n \n \n \n') + + fh.write(' \n \n \n
OperationNameTypeTTLValueSource
') + fh.write(change.__class__.__name__) + fh.write('') + fh.write(record.name) + fh.write('') + fh.write(record._type) + fh.write('') + fh.write(str(existing.ttl)) + fh.write('') + fh.write(_value_stringifier(existing, '
')) + fh.write('
') + fh.write(str(new.ttl)) + fh.write('') + fh.write(_value_stringifier(new, '
')) + fh.write('
') + fh.write(new.source.id) + fh.write('
Summary: ') + fh.write(str(plan)) + fh.write('
\n') + else: + fh.write('No changes were planned') diff --git a/tests/config/bad-plan-output-config.yaml b/tests/config/bad-plan-output-config.yaml new file mode 100644 index 0000000..f345f89 --- /dev/null +++ b/tests/config/bad-plan-output-config.yaml @@ -0,0 +1,7 @@ +manager: + plan_outputs: + 'bad': + class: octodns.provider.plan.PlanLogger + invalid: config +providers: {} +zones: {} diff --git a/tests/config/bad-plan-output-missing-class.yaml b/tests/config/bad-plan-output-missing-class.yaml new file mode 100644 index 0000000..71b1bd5 --- /dev/null +++ b/tests/config/bad-plan-output-missing-class.yaml @@ -0,0 +1,5 @@ +manager: + plan_outputs: + 'bad': {} +providers: {} +zones: {} diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 4db2103..ada54e5 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -83,6 +83,19 @@ class TestManager(TestCase): .sync(['unknown.target.']) self.assertTrue('unknown target' in ctx.exception.message) + def test_bad_plan_output_class(self): + with self.assertRaises(Exception) as ctx: + name = 'bad-plan-output-missing-class.yaml' + Manager(get_config_filename(name)).sync() + self.assertEquals('plan_output bad is missing class', + ctx.exception.message) + + def test_bad_plan_output_config(self): + with self.assertRaises(Exception) as ctx: + Manager(get_config_filename('bad-plan-output-config.yaml')).sync() + self.assertEqual('Incorrect plan_output config for bad', + ctx.exception.message) + def test_source_only_as_a_target(self): with self.assertRaises(Exception) as ctx: Manager(get_config_filename('unknown-provider.yaml')) \ diff --git a/tests/test_octodns_plan.py b/tests/test_octodns_plan.py new file mode 100644 index 0000000..ea35243 --- /dev/null +++ b/tests/test_octodns_plan.py @@ -0,0 +1,92 @@ +# +# +# + +from __future__ import absolute_import, division, print_function, \ + unicode_literals + +from StringIO import StringIO +from logging import getLogger +from unittest import TestCase + +from octodns.provider.plan import Plan, PlanHtml, PlanLogger, PlanMarkdown +from octodns.record import Create, Delete, Record, Update +from octodns.zone import Zone + +from helpers import SimpleProvider + + +class TestPlanLogger(TestCase): + + def test_invalid_level(self): + with self.assertRaises(Exception) as ctx: + PlanLogger('invalid', 'not-a-level') + self.assertEquals('Unsupported level: not-a-level', + ctx.exception.message) + + +simple = SimpleProvider() +zone = Zone('unit.tests.', []) +existing = Record.new(zone, 'a', { + 'ttl': 300, + 'type': 'A', + # This matches the zone data above, one to swap, one to leave + 'values': ['1.1.1.1', '2.2.2.2'], +}) +new = Record.new(zone, 'a', { + 'geo': { + 'AF': ['5.5.5.5'], + 'NA-US': ['6.6.6.6'] + }, + 'ttl': 300, + 'type': 'A', + # This leaves one, swaps ones, and adds one + 'values': ['2.2.2.2', '3.3.3.3', '4.4.4.4'], +}, simple) +create = Create(Record.new(zone, 'b', { + 'ttl': 60, + 'type': 'CNAME', + 'value': 'foo.unit.tests.' +}, simple)) +update = Update(existing, new) +delete = Delete(new) +changes = [create, delete, update] +plans = [ + (simple, Plan(zone, zone, changes)), + (simple, Plan(zone, zone, changes)), +] + + +class TestPlanHtml(TestCase): + log = getLogger('TestPlanHtml') + + def test_empty(self): + out = StringIO() + PlanHtml('html').run([], fh=out) + self.assertEquals('No changes were planned', out.getvalue()) + + def test_simple(self): + out = StringIO() + PlanHtml('html').run(plans, fh=out) + out = out.getvalue() + self.assertTrue(' Summary: Creates=1, Updates=1, ' + 'Deletes=1, Existing Records=0' in out) + + +class TestPlanMarkdown(TestCase): + log = getLogger('TestPlanMarkdown') + + def test_empty(self): + out = StringIO() + PlanMarkdown('markdown').run([], fh=out) + self.assertEquals('## No changes were planned\n', out.getvalue()) + + def test_simple(self): + out = StringIO() + PlanMarkdown('markdown').run(plans, fh=out) + out = out.getvalue() + self.assertTrue('## unit.tests.' in out) + self.assertTrue('Create | b | CNAME | 60 | foo.unit.tests.' in out) + self.assertTrue('Update | a | A | 300 | 1.1.1.1;' in out) + self.assertTrue('NA-US: 6.6.6.6 | test' in out) + self.assertTrue('Delete | a | A | 300 | 2.2.2.2;' in out) diff --git a/tests/test_octodns_provider_base.py b/tests/test_octodns_provider_base.py index 1bf3fd7..472b008 100644 --- a/tests/test_octodns_provider_base.py +++ b/tests/test_octodns_provider_base.py @@ -9,7 +9,8 @@ from logging import getLogger from unittest import TestCase from octodns.record import Create, Delete, Record, Update -from octodns.provider.base import BaseProvider, Plan, UnsafePlan +from octodns.provider.base import BaseProvider +from octodns.provider.plan import Plan, UnsafePlan from octodns.zone import Zone