Merge pull request #898 from octodns/dump-support-output-provider

Implement a sketch of --output-provider support for dump

octodns/cmds/dump.py

@@ -28,6 +28,12 @@ def main():
         help='The directory into which the results will be '
         'written (Note: will overwrite existing files)',
     )
+    parser.add_argument(
+        '--output-provider',
+        required=False,
+        help='The configured provider to use when dumping '
+        'records. Must support copy() and directory',
+    )
     parser.add_argument(
         '--lenient',
         action='store_true',
@@ -47,7 +53,12 @@ def main():
 
     manager = Manager(args.config_file)
     manager.dump(
-        args.zone, args.output_dir, args.lenient, args.split, *args.source
+        zone=args.zone,
+        output_dir=args.output_dir,
+        output_provider=args.output_provider,
+        lenient=args.lenient,
+        split=args.split,
+        sources=args.source,
     )
 
 

octodns/manager.py

@@ -638,25 +638,75 @@ class Manager(object):
 
         return zb.changes(za, _AggregateTarget(a + b))
 
-    def dump(self, zone, output_dir, lenient, split, source, *sources):
+    def dump(
+        self,
+        zone,
+        output_dir,
+        sources,
+        lenient=False,
+        split=False,
+        output_provider=None,
+    ):
         '''
         Dump zone data from the specified source
         '''
-        self.log.info('dump: zone=%s, sources=%s', zone, sources)
-
-        # We broke out source to force at least one to be passed, add it to any
-        # others we got.
-        sources = [source] + list(sources)
+        self.log.info(
+            'dump: zone=%s, output_dir=%s, output_provider=%s, '
+            'lenient=%s, split=%s, sources=%s',
+            zone,
+            output_dir,
+            output_provider,
+            lenient,
+            split,
+            sources,
+        )
 
         try:
             sources = [self.providers[s] for s in sources]
         except KeyError as e:
             raise ManagerException(f'Unknown source: {e.args[0]}')
 
-        clz = YamlProvider
-        if split:
-            clz = SplitYamlProvider
-        target = clz('dump', output_dir)
+        if output_provider:
+            self.log.info(
+                'dump: using specified output_provider=%s', output_provider
+            )
+            try:
+                target = self.providers[output_provider]
+            except KeyError as e:
+                raise ManagerException(f'Unknown output_provider: {e.args[0]}')
+            # The chosen output provider has to support a directory property so
+            # that we can tell it where the user has requested the dumped files
+            # to reside.
+            if not hasattr(target, 'directory'):
+                msg = (
+                    f'output_provider={output_provider}, does not support '
+                    'directory property'
+                )
+                raise ManagerException(msg)
+            if target.directory != output_dir:
+                # If the requested target doesn't match what's configured in
+                # the chosen provider then we'll need to set it. Before doing
+                # that we make a copy of the provider so that it can remain
+                # unchanged and potentially be used as a source, e.g. copying
+                # from one yaml to another
+                if not hasattr(target, 'copy'):
+                    msg = (
+                        f'output_provider={output_provider}, does not '
+                        'support copy method'
+                    )
+                    raise ManagerException(msg)
+                target = target.copy()
+                self.log.info(
+                    'dump: setting directory of output_provider ' 'copy to %s',
+                    output_dir,
+                )
+                target.directory = output_dir
+        else:
+            self.log.info('dump: using custom YamlProvider')
+            clz = YamlProvider
+            if split:
+                clz = SplitYamlProvider
+            target = clz('dump', output_dir)
 
         zone = Zone(zone, self.configured_sub_zones(zone))
         for source in sources:

octodns/provider/yaml.py

@@ -161,6 +161,12 @@ class YamlProvider(BaseProvider):
         self.populate_should_replace = populate_should_replace
         self.supports_root_ns = supports_root_ns
 
+    def copy(self):
+        args = dict(self.__dict__)
+        args['id'] = f'{args["id"]}-copy'
+        del args['log']
+        return self.__class__(**args)
+
     @property
     def SUPPORTS_ROOT_NS(self):
         return self.supports_root_ns

tests/config/simple.yaml

@@ -8,12 +8,13 @@ providers:
   dump:
     class: octodns.provider.yaml.YamlProvider
     directory: env/YAML_TMP_DIR
+    default_ttl: 999
     supports_root_ns: False
   # This is sort of ugly, but it shouldn't hurt anything. It'll just write out
   # the target file twice where it and dump are both used
   dump2:
     class: octodns.provider.yaml.YamlProvider
-    directory: env/YAML_TMP_DIR
+    directory: env/YAML_TMP_DIR2
     supports_root_ns: False
   simple:
     class: helpers.SimpleProvider

tests/test_octodns_manager.py

@@ -10,7 +10,7 @@ from __future__ import (
 )
 
 from os import environ
-from os.path import dirname, join
+from os.path import dirname, isfile, join
 
 from octodns import __VERSION__
 from octodns.manager import (
@@ -130,6 +130,7 @@ class TestManager(TestCase):
     def test_always_dry_run(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
             tc = Manager(get_config_filename('always-dry-run.yaml')).sync(
                 dry_run=False
             )
@@ -139,6 +140,7 @@ class TestManager(TestCase):
     def test_simple(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
             tc = Manager(get_config_filename('simple.yaml')).sync(dry_run=False)
             self.assertEqual(28, tc)
 
@@ -183,6 +185,7 @@ class TestManager(TestCase):
     def test_eligible_sources(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
             # Only allow a target that doesn't exist
             tc = Manager(get_config_filename('simple.yaml')).sync(
                 eligible_sources=['foo']
@@ -192,6 +195,7 @@ class TestManager(TestCase):
     def test_eligible_targets(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
             # Only allow a target that doesn't exist
             tc = Manager(get_config_filename('simple.yaml')).sync(
                 eligible_targets=['foo']
@@ -201,6 +205,7 @@ class TestManager(TestCase):
     def test_aliases(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
             # Alias zones with a valid target.
             tc = Manager(get_config_filename('simple-alias-zone.yaml')).sync()
             self.assertEqual(0, tc)
@@ -239,6 +244,7 @@ class TestManager(TestCase):
     def test_compare(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
             manager = Manager(get_config_filename('simple.yaml'))
 
             # make sure this was pulled in from the config
@@ -319,49 +325,157 @@ class TestManager(TestCase):
     def test_dump(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
             manager = Manager(get_config_filename('simple.yaml'))
 
             with self.assertRaises(ManagerException) as ctx:
                 manager.dump(
-                    'unit.tests.', tmpdir.dirname, False, False, 'nope'
+                    zone='unit.tests.',
+                    output_dir=tmpdir.dirname,
+                    split=True,
+                    sources=['nope'],
                 )
             self.assertEqual('Unknown source: nope', str(ctx.exception))
 
-            manager.dump('unit.tests.', tmpdir.dirname, False, False, 'in')
+            manager.dump(
+                zone='unit.tests.',
+                output_dir=tmpdir.dirname,
+                split=True,
+                sources=['in'],
+            )
 
             # make sure this fails with an IOError and not a KeyError when
             # trying to find sub zones
             with self.assertRaises(IOError):
                 manager.dump(
-                    'unknown.zone.', tmpdir.dirname, False, False, 'in'
+                    zone='unknown.zone.',
+                    output_dir=tmpdir.dirname,
+                    split=True,
+                    sources=['in'],
                 )
 
     def test_dump_empty(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
             manager = Manager(get_config_filename('simple.yaml'))
 
-            manager.dump('empty.', tmpdir.dirname, False, False, 'in')
+            manager.dump(
+                zone='empty.', output_dir=tmpdir.dirname, sources=['in']
+            )
 
             with open(join(tmpdir.dirname, 'empty.yaml')) as fh:
                 data = safe_load(fh, False)
                 self.assertFalse(data)
 
+    def test_dump_output_provider(self):
+        with TemporaryDirectory() as tmpdir:
+            environ['YAML_TMP_DIR'] = tmpdir.dirname
+            # this time we'll use seperate tmp dirs
+            with TemporaryDirectory() as tmpdir2:
+                environ['YAML_TMP_DIR2'] = tmpdir2.dirname
+                manager = Manager(get_config_filename('simple.yaml'))
+
+                # we're going to tell it to use dump2 to do the dumping, but a
+                # copy should be made and directory set to tmpdir.dirname
+                # rather than 2's tmpdir2.dirname
+                manager.dump(
+                    zone='unit.tests.',
+                    output_dir=tmpdir.dirname,
+                    output_provider='dump2',
+                    sources=['in'],
+                )
+
+                self.assertTrue(isfile(join(tmpdir.dirname, 'unit.tests.yaml')))
+                self.assertFalse(
+                    isfile(join(tmpdir2.dirname, 'unit.tests.yaml'))
+                )
+
+                # let's run that again, this time telling it to use tmpdir2 and
+                # dump2 which should allow it to skip the copying
+                manager.dump(
+                    zone='unit.tests.',
+                    output_dir=tmpdir2.dirname,
+                    output_provider='dump2',
+                    sources=['in'],
+                )
+                self.assertTrue(
+                    isfile(join(tmpdir2.dirname, 'unit.tests.yaml'))
+                )
+
+                # tell it to use an output_provider that doesn't exist
+                with self.assertRaises(ManagerException) as ctx:
+                    manager.dump(
+                        zone='unit.tests.',
+                        output_dir=tmpdir.dirname,
+                        output_provider='nope',
+                        sources=['in'],
+                    )
+                self.assertEqual(
+                    'Unknown output_provider: nope', str(ctx.exception)
+                )
+
+                # tell it to use an output_provider that doesn't support
+                # directory
+                with self.assertRaises(ManagerException) as ctx:
+                    manager.dump(
+                        zone='unit.tests.',
+                        output_dir=tmpdir.dirname,
+                        output_provider='simple',
+                        sources=['in'],
+                    )
+                self.assertEqual(
+                    'output_provider=simple, does not support '
+                    'directory property',
+                    str(ctx.exception),
+                )
+
+                # hack a directory property onto the simple provider so that
+                # it'll pass that check and fail the copy one instead
+                manager.providers['simple'].directory = 42
+                with self.assertRaises(ManagerException) as ctx:
+                    manager.dump(
+                        zone='unit.tests.',
+                        output_dir=tmpdir.dirname,
+                        output_provider='simple',
+                        sources=['in'],
+                    )
+                self.assertEqual(
+                    'output_provider=simple, does not support ' 'copy method',
+                    str(ctx.exception),
+                )
+
     def test_dump_split(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
             manager = Manager(get_config_filename('simple-split.yaml'))
 
             with self.assertRaises(ManagerException) as ctx:
-                manager.dump('unit.tests.', tmpdir.dirname, False, True, 'nope')
+                manager.dump(
+                    zone='unit.tests.',
+                    output_dir=tmpdir.dirname,
+                    split=True,
+                    sources=['nope'],
+                )
             self.assertEqual('Unknown source: nope', str(ctx.exception))
 
-            manager.dump('unit.tests.', tmpdir.dirname, False, True, 'in')
+            manager.dump(
+                zone='unit.tests.',
+                output_dir=tmpdir.dirname,
+                split=True,
+                sources=['in'],
+            )
 
             # make sure this fails with an OSError and not a KeyError when
             # trying to find sub zones
             with self.assertRaises(OSError):
-                manager.dump('unknown.zone.', tmpdir.dirname, False, True, 'in')
+                manager.dump(
+                    zone='unknown.zone.',
+                    output_dir=tmpdir.dirname,
+                    split=True,
+                    sources=['in'],
+                )
 
     def test_validate_configs(self):
         Manager(get_config_filename('simple-validate.yaml')).validate_configs()
@@ -419,6 +533,7 @@ class TestManager(TestCase):
     def test_populate_lenient_fallback(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
             # Only allow a target that doesn't exist
             manager = Manager(get_config_filename('simple.yaml'))
 
@@ -441,6 +556,7 @@ class TestManager(TestCase):
     def test_plan_processors_fallback(self):
         with TemporaryDirectory() as tmpdir:
             environ['YAML_TMP_DIR'] = tmpdir.dirname
+            environ['YAML_TMP_DIR2'] = tmpdir.dirname
             # Only allow a target that doesn't exist
             manager = Manager(get_config_filename('simple.yaml'))
 

tests/test_octodns_provider_yaml.py

@@ -444,6 +444,46 @@ class TestSplitYamlProvider(TestCase):
             str(ctx.exception),
         )
 
+    def test_copy(self):
+        # going to put some sentinal values in here to ensure, these aren't
+        # valid, but we shouldn't hit any code that cares during this test
+        source = YamlProvider(
+            'test',
+            42,
+            default_ttl=43,
+            enforce_order=44,
+            populate_should_replace=45,
+            supports_root_ns=46,
+        )
+        copy = source.copy()
+        self.assertEqual(source.directory, copy.directory)
+        self.assertEqual(source.default_ttl, copy.default_ttl)
+        self.assertEqual(source.enforce_order, copy.enforce_order)
+        self.assertEqual(
+            source.populate_should_replace, copy.populate_should_replace
+        )
+        self.assertEqual(source.supports_root_ns, copy.supports_root_ns)
+
+        # same for split
+        source = SplitYamlProvider(
+            'test',
+            42,
+            extension=42.5,
+            default_ttl=43,
+            enforce_order=44,
+            populate_should_replace=45,
+            supports_root_ns=46,
+        )
+        copy = source.copy()
+        self.assertEqual(source.directory, copy.directory)
+        self.assertEqual(source.extension, copy.extension)
+        self.assertEqual(source.default_ttl, copy.default_ttl)
+        self.assertEqual(source.enforce_order, copy.enforce_order)
+        self.assertEqual(
+            source.populate_should_replace, copy.populate_should_replace
+        )
+        self.assertEqual(source.supports_root_ns, copy.supports_root_ns)
+
 
 class TestOverridingYamlProvider(TestCase):
     def test_provider(self):