diff --git a/octodns/cmds/dump.py b/octodns/cmds/dump.py index 41396a0..8d30913 100755 --- a/octodns/cmds/dump.py +++ b/octodns/cmds/dump.py @@ -28,6 +28,12 @@ def main(): help='The directory into which the results will be ' 'written (Note: will overwrite existing files)', ) + parser.add_argument( + '--output-provider', + required=False, + help='The configured provider to use when dumping ' + 'records. Must support copy() and directory', + ) parser.add_argument( '--lenient', action='store_true', @@ -47,7 +53,12 @@ def main(): manager = Manager(args.config_file) manager.dump( - args.zone, args.output_dir, args.lenient, args.split, *args.source + zone=args.zone, + output_dir=args.output_dir, + output_provider=args.output_provider, + lenient=args.lenient, + split=args.split, + sources=args.source, ) diff --git a/octodns/manager.py b/octodns/manager.py index d68e817..0faa0bf 100644 --- a/octodns/manager.py +++ b/octodns/manager.py @@ -638,25 +638,75 @@ class Manager(object): return zb.changes(za, _AggregateTarget(a + b)) - def dump(self, zone, output_dir, lenient, split, source, *sources): + def dump( + self, + zone, + output_dir, + sources, + lenient=False, + split=False, + output_provider=None, + ): ''' Dump zone data from the specified source ''' - self.log.info('dump: zone=%s, sources=%s', zone, sources) - - # We broke out source to force at least one to be passed, add it to any - # others we got. - sources = [source] + list(sources) + self.log.info( + 'dump: zone=%s, output_dir=%s, output_provider=%s, ' + 'lenient=%s, split=%s, sources=%s', + zone, + output_dir, + output_provider, + lenient, + split, + sources, + ) try: sources = [self.providers[s] for s in sources] except KeyError as e: raise ManagerException(f'Unknown source: {e.args[0]}') - clz = YamlProvider - if split: - clz = SplitYamlProvider - target = clz('dump', output_dir) + if output_provider: + self.log.info( + 'dump: using specified output_provider=%s', output_provider + ) + try: + target = self.providers[output_provider] + except KeyError as e: + raise ManagerException(f'Unknown output_provider: {e.args[0]}') + # The chosen output provider has to support a directory property so + # that we can tell it where the user has requested the dumped files + # to reside. + if not hasattr(target, 'directory'): + msg = ( + f'output_provider={output_provider}, does not support ' + 'directory property' + ) + raise ManagerException(msg) + if target.directory != output_dir: + # If the requested target doesn't match what's configured in + # the chosen provider then we'll need to set it. Before doing + # that we make a copy of the provider so that it can remain + # unchanged and potentially be used as a source, e.g. copying + # from one yaml to another + if not hasattr(target, 'copy'): + msg = ( + f'output_provider={output_provider}, does not ' + 'support copy method' + ) + raise ManagerException(msg) + target = target.copy() + self.log.info( + 'dump: setting directory of output_provider ' 'copy to %s', + output_dir, + ) + target.directory = output_dir + else: + self.log.info('dump: using custom YamlProvider') + clz = YamlProvider + if split: + clz = SplitYamlProvider + target = clz('dump', output_dir) zone = Zone(zone, self.configured_sub_zones(zone)) for source in sources: diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py index 883f316..9ad0934 100644 --- a/octodns/provider/yaml.py +++ b/octodns/provider/yaml.py @@ -161,6 +161,12 @@ class YamlProvider(BaseProvider): self.populate_should_replace = populate_should_replace self.supports_root_ns = supports_root_ns + def copy(self): + args = dict(self.__dict__) + args['id'] = f'{args["id"]}-copy' + del args['log'] + return self.__class__(**args) + @property def SUPPORTS_ROOT_NS(self): return self.supports_root_ns diff --git a/tests/config/simple.yaml b/tests/config/simple.yaml index 4cd19c8..5040298 100644 --- a/tests/config/simple.yaml +++ b/tests/config/simple.yaml @@ -8,12 +8,13 @@ providers: dump: class: octodns.provider.yaml.YamlProvider directory: env/YAML_TMP_DIR + default_ttl: 999 supports_root_ns: False # This is sort of ugly, but it shouldn't hurt anything. It'll just write out # the target file twice where it and dump are both used dump2: class: octodns.provider.yaml.YamlProvider - directory: env/YAML_TMP_DIR + directory: env/YAML_TMP_DIR2 supports_root_ns: False simple: class: helpers.SimpleProvider diff --git a/tests/test_octodns_manager.py b/tests/test_octodns_manager.py index 44cf3e4..ab987d2 100644 --- a/tests/test_octodns_manager.py +++ b/tests/test_octodns_manager.py @@ -10,7 +10,7 @@ from __future__ import ( ) from os import environ -from os.path import dirname, join +from os.path import dirname, isfile, join from octodns import __VERSION__ from octodns.manager import ( @@ -130,6 +130,7 @@ class TestManager(TestCase): def test_always_dry_run(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname tc = Manager(get_config_filename('always-dry-run.yaml')).sync( dry_run=False ) @@ -139,6 +140,7 @@ class TestManager(TestCase): def test_simple(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname tc = Manager(get_config_filename('simple.yaml')).sync(dry_run=False) self.assertEqual(28, tc) @@ -183,6 +185,7 @@ class TestManager(TestCase): def test_eligible_sources(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname # Only allow a target that doesn't exist tc = Manager(get_config_filename('simple.yaml')).sync( eligible_sources=['foo'] @@ -192,6 +195,7 @@ class TestManager(TestCase): def test_eligible_targets(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname # Only allow a target that doesn't exist tc = Manager(get_config_filename('simple.yaml')).sync( eligible_targets=['foo'] @@ -201,6 +205,7 @@ class TestManager(TestCase): def test_aliases(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname # Alias zones with a valid target. tc = Manager(get_config_filename('simple-alias-zone.yaml')).sync() self.assertEqual(0, tc) @@ -239,6 +244,7 @@ class TestManager(TestCase): def test_compare(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname manager = Manager(get_config_filename('simple.yaml')) # make sure this was pulled in from the config @@ -319,49 +325,157 @@ class TestManager(TestCase): def test_dump(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname manager = Manager(get_config_filename('simple.yaml')) with self.assertRaises(ManagerException) as ctx: manager.dump( - 'unit.tests.', tmpdir.dirname, False, False, 'nope' + zone='unit.tests.', + output_dir=tmpdir.dirname, + split=True, + sources=['nope'], ) self.assertEqual('Unknown source: nope', str(ctx.exception)) - manager.dump('unit.tests.', tmpdir.dirname, False, False, 'in') + manager.dump( + zone='unit.tests.', + output_dir=tmpdir.dirname, + split=True, + sources=['in'], + ) # make sure this fails with an IOError and not a KeyError when # trying to find sub zones with self.assertRaises(IOError): manager.dump( - 'unknown.zone.', tmpdir.dirname, False, False, 'in' + zone='unknown.zone.', + output_dir=tmpdir.dirname, + split=True, + sources=['in'], ) def test_dump_empty(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname manager = Manager(get_config_filename('simple.yaml')) - manager.dump('empty.', tmpdir.dirname, False, False, 'in') + manager.dump( + zone='empty.', output_dir=tmpdir.dirname, sources=['in'] + ) with open(join(tmpdir.dirname, 'empty.yaml')) as fh: data = safe_load(fh, False) self.assertFalse(data) + def test_dump_output_provider(self): + with TemporaryDirectory() as tmpdir: + environ['YAML_TMP_DIR'] = tmpdir.dirname + # this time we'll use seperate tmp dirs + with TemporaryDirectory() as tmpdir2: + environ['YAML_TMP_DIR2'] = tmpdir2.dirname + manager = Manager(get_config_filename('simple.yaml')) + + # we're going to tell it to use dump2 to do the dumping, but a + # copy should be made and directory set to tmpdir.dirname + # rather than 2's tmpdir2.dirname + manager.dump( + zone='unit.tests.', + output_dir=tmpdir.dirname, + output_provider='dump2', + sources=['in'], + ) + + self.assertTrue(isfile(join(tmpdir.dirname, 'unit.tests.yaml'))) + self.assertFalse( + isfile(join(tmpdir2.dirname, 'unit.tests.yaml')) + ) + + # let's run that again, this time telling it to use tmpdir2 and + # dump2 which should allow it to skip the copying + manager.dump( + zone='unit.tests.', + output_dir=tmpdir2.dirname, + output_provider='dump2', + sources=['in'], + ) + self.assertTrue( + isfile(join(tmpdir2.dirname, 'unit.tests.yaml')) + ) + + # tell it to use an output_provider that doesn't exist + with self.assertRaises(ManagerException) as ctx: + manager.dump( + zone='unit.tests.', + output_dir=tmpdir.dirname, + output_provider='nope', + sources=['in'], + ) + self.assertEqual( + 'Unknown output_provider: nope', str(ctx.exception) + ) + + # tell it to use an output_provider that doesn't support + # directory + with self.assertRaises(ManagerException) as ctx: + manager.dump( + zone='unit.tests.', + output_dir=tmpdir.dirname, + output_provider='simple', + sources=['in'], + ) + self.assertEqual( + 'output_provider=simple, does not support ' + 'directory property', + str(ctx.exception), + ) + + # hack a directory property onto the simple provider so that + # it'll pass that check and fail the copy one instead + manager.providers['simple'].directory = 42 + with self.assertRaises(ManagerException) as ctx: + manager.dump( + zone='unit.tests.', + output_dir=tmpdir.dirname, + output_provider='simple', + sources=['in'], + ) + self.assertEqual( + 'output_provider=simple, does not support ' 'copy method', + str(ctx.exception), + ) + def test_dump_split(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname manager = Manager(get_config_filename('simple-split.yaml')) with self.assertRaises(ManagerException) as ctx: - manager.dump('unit.tests.', tmpdir.dirname, False, True, 'nope') + manager.dump( + zone='unit.tests.', + output_dir=tmpdir.dirname, + split=True, + sources=['nope'], + ) self.assertEqual('Unknown source: nope', str(ctx.exception)) - manager.dump('unit.tests.', tmpdir.dirname, False, True, 'in') + manager.dump( + zone='unit.tests.', + output_dir=tmpdir.dirname, + split=True, + sources=['in'], + ) # make sure this fails with an OSError and not a KeyError when # trying to find sub zones with self.assertRaises(OSError): - manager.dump('unknown.zone.', tmpdir.dirname, False, True, 'in') + manager.dump( + zone='unknown.zone.', + output_dir=tmpdir.dirname, + split=True, + sources=['in'], + ) def test_validate_configs(self): Manager(get_config_filename('simple-validate.yaml')).validate_configs() @@ -419,6 +533,7 @@ class TestManager(TestCase): def test_populate_lenient_fallback(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname # Only allow a target that doesn't exist manager = Manager(get_config_filename('simple.yaml')) @@ -441,6 +556,7 @@ class TestManager(TestCase): def test_plan_processors_fallback(self): with TemporaryDirectory() as tmpdir: environ['YAML_TMP_DIR'] = tmpdir.dirname + environ['YAML_TMP_DIR2'] = tmpdir.dirname # Only allow a target that doesn't exist manager = Manager(get_config_filename('simple.yaml')) diff --git a/tests/test_octodns_provider_yaml.py b/tests/test_octodns_provider_yaml.py index f332eae..e2f55c2 100644 --- a/tests/test_octodns_provider_yaml.py +++ b/tests/test_octodns_provider_yaml.py @@ -444,6 +444,46 @@ class TestSplitYamlProvider(TestCase): str(ctx.exception), ) + def test_copy(self): + # going to put some sentinal values in here to ensure, these aren't + # valid, but we shouldn't hit any code that cares during this test + source = YamlProvider( + 'test', + 42, + default_ttl=43, + enforce_order=44, + populate_should_replace=45, + supports_root_ns=46, + ) + copy = source.copy() + self.assertEqual(source.directory, copy.directory) + self.assertEqual(source.default_ttl, copy.default_ttl) + self.assertEqual(source.enforce_order, copy.enforce_order) + self.assertEqual( + source.populate_should_replace, copy.populate_should_replace + ) + self.assertEqual(source.supports_root_ns, copy.supports_root_ns) + + # same for split + source = SplitYamlProvider( + 'test', + 42, + extension=42.5, + default_ttl=43, + enforce_order=44, + populate_should_replace=45, + supports_root_ns=46, + ) + copy = source.copy() + self.assertEqual(source.directory, copy.directory) + self.assertEqual(source.extension, copy.extension) + self.assertEqual(source.default_ttl, copy.default_ttl) + self.assertEqual(source.enforce_order, copy.enforce_order) + self.assertEqual( + source.populate_should_replace, copy.populate_should_replace + ) + self.assertEqual(source.supports_root_ns, copy.supports_root_ns) + class TestOverridingYamlProvider(TestCase): def test_provider(self):