Merge pull request #1044 from octodns/yaml-provider-refactor

Deprecate SplitYamlProvider, subsume functionality into YamlProvider w/a refactor
2024-05-11 05:55:00 +00:00 · 2023-08-31 11:50:10 -06:00
parent e9d6e6d11f 93b397cbb2
commit 8237aa955d
5 changed files with 397 additions and 179 deletions
@@ -5,6 +5,12 @@
 * New dynamic zone config support that allows wildcard entries in the octoDNS
  config to be expanded by the source provider(s). See
  [Dynamic Zone Config](/README.md#dynamic-zone-config) for more information.
+* SplitYamlProvider has been deprecated and will be removed in 2.0. YamlProvider
+  now includes the ability to process split zones when configured to do so and
+  allows for more flexibility in how things are laid out than was previously
+  possible. This includes the ability to split some zones and not others and
+  even to have partially split zones with some records in the primary zone YAML
+  and others in a split directory. See YamlProvider documentation for more info.

 #### Stuff

@@ -19,18 +19,70 @@ class YamlProvider(BaseProvider):

    config:
        class: octodns.provider.yaml.YamlProvider
-        # The location of yaml config files (required)
+
+        # The location of yaml config files. By default records are defined in a
+        # file named for the zone in this directory, the zone file, e.g.
+        # something.com.yaml.
+        # (required)
        directory: ./config
+
        # The ttl to use for records when not specified in the data
        # (optional, default 3600)
        default_ttl: 3600
-        # Whether or not to enforce sorting order on the yaml config
+
+        # Whether or not to enforce sorting order when loading yaml
        # (optional, default True)
        enforce_order: true
+
        # Whether duplicate records should replace rather than error
-        # (optiona, default False)
+        # (optional, default False)
        populate_should_replace: false

+        # The file extension used when loading split style zones, Null means
+        # disabled. When enabled the provider will search for zone records split
+        # across multiple YAML files in the directory with split_extension
+        # appended to the zone name, See "Split Details" below.
+        # split_extension should include the "."
+        # (optional, default null, "." is the recommended best practice when
+        # enabling)
+        split_extension: null
+
+        # When writing YAML records out to disk with split_extension enabled
+        # each record is written out into its own file with .yaml appended to
+        # the name of the record. The two exceptions are for the root and
+        # wildcard nodes. These records are written into a file named
+        # `$[zone.name].yaml`. If you would prefer this catchall file not be
+        # used `split_catchall` can be set to False to instead write those
+        # records out to `.yaml` and `*.yaml` respectively. Note that some
+        # operating systems may not allow files with those names.
+        # (optional, default True)
+        split_catchall: true
+
+        # Disable loading of the zone .yaml files.
+        # (optional, default False)
+        disable_zonefile: false
+
+    Split Details
+    -------------
+
+    All files are stored in a subdirectory matching the name of the zone
+    (including the trailing .) of the directory config. It is a recommended
+    best practice that the files be named RECORD.yaml, but all files are
+    sourced and processed ignoring the filenames so it is up to you how to
+    organize them.
+
+    With `split_extension: .` the directory structure for the zone github.com.
+    managed under directory "zones/" would look like:
+
+    zones/
+      github.com./
+        $github.com.yaml
+        www.yaml
+        ...
+
+    Overriding Values
+    -----------------
+
    Overriding values can be accomplished using multiple yaml providers in the
    `sources` list where subsequent providers have `populate_should_replace`
    set to `true`. An example use of this would be a zone that you want to push
@@ -98,7 +150,6 @@ class YamlProvider(BaseProvider):
    You can then sync our records eternally with `--config-file=external.yaml`
    and internally (with the custom overrides) with
    `--config-file=internal.yaml`
-
    '''

    SUPPORTS_GEO = True
@@ -107,6 +158,10 @@ class YamlProvider(BaseProvider):
    SUPPORTS_DYNAMIC_SUBNETS = True
    SUPPORTS_MULTIVALUE_PTR = True

+    # Any record name added to this set will be included in the catch-all file,
+    # instead of a file matching the record name.
+    CATCHALL_RECORD_NAMES = ('*', '')
+
    def __init__(
        self,
        id,
@@ -115,19 +170,25 @@ class YamlProvider(BaseProvider):
        enforce_order=True,
        populate_should_replace=False,
        supports_root_ns=True,
+        split_extension=False,
+        split_catchall=True,
+        disable_zonefile=False,
        *args,
        **kwargs,
    ):
        klass = self.__class__.__name__
        self.log = logging.getLogger(f'{klass}[{id}]')
        self.log.debug(
-            '__init__: id=%s, directory=%s, default_ttl=%d, '
-            'enforce_order=%d, populate_should_replace=%d',
+            '__init__: id=%s, directory=%s, default_ttl=%d, enforce_order=%d, populate_should_replace=%s, supports_root_ns=%s, split_extension=%s, split_catchall=%s, disable_zonefile=%s',
            id,
            directory,
            default_ttl,
            enforce_order,
            populate_should_replace,
+            supports_root_ns,
+            split_extension,
+            split_catchall,
+            disable_zonefile,
        )
        super().__init__(id, *args, **kwargs)
        self.directory = directory
@@ -135,12 +196,15 @@ class YamlProvider(BaseProvider):
        self.enforce_order = enforce_order
        self.populate_should_replace = populate_should_replace
        self.supports_root_ns = supports_root_ns
+        self.split_extension = split_extension
+        self.split_catchall = split_catchall
+        self.disable_zonefile = disable_zonefile

    def copy(self):
-        args = dict(self.__dict__)
-        args['id'] = f'{args["id"]}-copy'
-        del args['log']
-        return self.__class__(**args)
+        kwargs = dict(self.__dict__)
+        kwargs['id'] = f'{kwargs["id"]}-copy'
+        del kwargs['log']
+        return YamlProvider(**kwargs)

    @property
    def SUPPORTS(self):
@@ -162,6 +226,69 @@ class YamlProvider(BaseProvider):
    def SUPPORTS_ROOT_NS(self):
        return self.supports_root_ns

+    def list_zones(self):
+        self.log.debug('list_zones:')
+        zones = set()
+
+        extension = self.split_extension
+        if extension:
+            # we want to leave the .
+            trim = len(extension) - 1
+            self.log.debug(
+                'list_zones:   looking for split zones, trim=%d', trim
+            )
+            for dirname in listdir(self.directory):
+                not_ends_with = not dirname.endswith(extension)
+                not_dir = not isdir(join(self.directory, dirname))
+                if not_dir or not_ends_with:
+                    continue
+                if trim:
+                    dirname = dirname[:-trim]
+                zones.add(dirname)
+
+        if not self.disable_zonefile:
+            self.log.debug('list_zones:   looking for zone files')
+            for filename in listdir(self.directory):
+                not_ends_with = not filename.endswith('.yaml')
+                too_few_dots = filename.count('.') < 2
+                not_file = not isfile(join(self.directory, filename))
+                if not_file or not_ends_with or too_few_dots:
+                    continue
+                # trim off the yaml, leave the .
+                zones.add(filename[:-4])
+
+        return sorted(zones)
+
+    def _split_sources(self, zone):
+        ext = self.split_extension
+        utf8 = join(self.directory, f'{zone.decoded_name[:-1]}{ext}')
+        idna = join(self.directory, f'{zone.name[:-1]}{ext}')
+        directory = None
+        if isdir(utf8):
+            if utf8 != idna and isdir(idna):
+                raise ProviderException(
+                    f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}'
+                )
+            directory = utf8
+        else:
+            directory = idna
+
+        for filename in listdir(directory):
+            if filename.endswith('.yaml'):
+                yield join(directory, filename)
+
+    def _zone_sources(self, zone):
+        utf8 = join(self.directory, f'{zone.decoded_name}yaml')
+        idna = join(self.directory, f'{zone.name}yaml')
+        if isfile(utf8):
+            if utf8 != idna and isfile(idna):
+                raise ProviderException(
+                    f'Both UTF-8 "{utf8}" and IDNA "{idna}" exist for {zone.decoded_name}'
+                )
+            return utf8
+
+        return idna
+
    def _populate_from_file(self, filename, zone, lenient):
        with open(filename, 'r') as fh:
            yaml_data = safe_load(fh, enforce_order=self.enforce_order)
@@ -184,18 +311,6 @@ class YamlProvider(BaseProvider):
                '_populate_from_file: successfully loaded "%s"', filename
            )

-    def get_filenames(self, zone):
-        return (
-            join(self.directory, f'{zone.decoded_name}yaml'),
-            join(self.directory, f'{zone.name}yaml'),
-        )
-
-    def list_zones(self):
-        for filename in listdir(self.directory):
-            if not filename.endswith('.yaml') or filename.count('.') < 2:
-                continue
-            yield filename[:-4]
-
    def populate(self, zone, target=False, lenient=False):
        self.log.debug(
            'populate: name=%s, target=%s, lenient=%s',
@@ -210,23 +325,21 @@ class YamlProvider(BaseProvider):
            return False

        before = len(zone.records)
-        utf8_filename, idna_filename = self.get_filenames(zone)

-        # we prefer utf8
-        if isfile(utf8_filename):
-            if utf8_filename != idna_filename and isfile(idna_filename):
-                raise ProviderException(
-                    f'Both UTF-8 "{utf8_filename}" and IDNA "{idna_filename}" exist for {zone.decoded_name}'
-                )
-            filename = utf8_filename
-        else:
-            self.log.warning(
-                'populate: "%s" does not exist, falling back to try idna version "%s"',
-                utf8_filename,
-                idna_filename,
-            )
-            filename = idna_filename
-        self._populate_from_file(filename, zone, lenient)
+        sources = []
+
+        split_extension = self.split_extension
+        if split_extension:
+            sources.extend(self._split_sources(zone))
+
+        if not self.disable_zonefile:
+            sources.append(self._zone_sources(zone))
+
+        # determinstically order our sources
+        sources.sort()
+
+        for source in sources:
+            self._populate_from_file(source, zone, lenient)

        self.log.info(
            'populate:   found %s records, exists=False',
@@ -264,123 +377,75 @@ class YamlProvider(BaseProvider):
                data[k] = data[k][0]

        if not isdir(self.directory):
+            self.log.debug('_apply: creating directory=%s', self.directory)
            makedirs(self.directory)

-        self._do_apply(desired, data)
+        if self.split_extension:
+            # we're going to do split files
+            decoded_name = desired.decoded_name[:-1]
+            directory = join(
+                self.directory, f'{decoded_name}{self.split_extension}'
+            )

-    def _do_apply(self, desired, data):
-        filename = join(self.directory, f'{desired.decoded_name}yaml')
-        self.log.debug('_apply:   writing filename=%s', filename)
-        with open(filename, 'w') as fh:
-            safe_dump(dict(data), fh, allow_unicode=True)
+            if not isdir(directory):
+                self.log.debug('_apply: creating split directory=%s', directory)
+                makedirs(directory)

+            catchall = {}
+            for record, config in data.items():
+                if self.split_catchall and record in self.CATCHALL_RECORD_NAMES:
+                    catchall[record] = config
+                    continue
+                filename = join(directory, f'{record}.yaml')
+                self.log.debug('_apply:   writing filename=%s', filename)

-def _list_all_yaml_files(directory):
-    yaml_files = set()
-    for f in listdir(directory):
-        filename = join(directory, f)
-        if f.endswith('.yaml') and isfile(filename):
-            yaml_files.add(filename)
-    return list(yaml_files)
+                with open(filename, 'w') as fh:
+                    record_data = {record: config}
+                    safe_dump(record_data, fh)
+
+            if catchall:
+                # Scrub the trailing . to make filenames more sane.
+                filename = join(directory, f'${decoded_name}.yaml')
+                self.log.debug(
+                    '_apply:   writing catchall filename=%s', filename
+                )
+                with open(filename, 'w') as fh:
+                    safe_dump(catchall, fh)
+
+        else:
+            # single large file
+            filename = join(self.directory, f'{desired.decoded_name}yaml')
+            self.log.debug('_apply:   writing filename=%s', filename)
+            with open(filename, 'w') as fh:
+                safe_dump(dict(data), fh, allow_unicode=True)


 class SplitYamlProvider(YamlProvider):
    '''
-    Core provider for records configured in multiple YAML files on disk.
+    DEPRECATED: Use YamlProvider with the split_extension parameter instead.

-    Behaves mostly similarly to YamlConfig, but interacts with multiple YAML
-    files, instead of a single monolitic one. All files are stored in a
-    subdirectory matching the name of the zone (including the trailing .) of
-    the directory config. The files are named RECORD.yaml, except for any
-    record which cannot be represented easily as a file; these are stored in
-    the catchall file, which is a YAML file the zone name, prepended with '$'.
-    For example, a zone, 'github.com.' would have a catch-all file named
-    '$github.com.yaml'.
+    When migrating the following configuration options would result in the same
+    behavior as SplitYamlProvider

-    A full directory structure for the zone github.com. managed under directory
-    "zones/" would be:
+       config:
+         class: octodns.provider.yaml.YamlProvider
+         # extension is configured as split_extension
+         split_extension: .
+         split_catchall: true
+         disable_zonefile: true

-    zones/
-      github.com./
-        $github.com.yaml
-        www.yaml
-        ...
-
-    config:
-        class: octodns.provider.yaml.SplitYamlProvider
-        # The location of yaml config files (required)
-        directory: ./config
-        # The ttl to use for records when not specified in the data
-        # (optional, default 3600)
-        default_ttl: 3600
-        # Whether or not to enforce sorting order on the yaml config
-        # (optional, default True)
-        enforce_order: True
+    TO BE REMOVED: 2.0
    '''

-    # Any record name added to this set will be included in the catch-all file,
-    # instead of a file matching the record name.
-    CATCHALL_RECORD_NAMES = ('*', '')
-
-    def __init__(self, id, directory, extension='.', *args, **kwargs):
+    def __init__(self, id, directory, *args, extension='.', **kwargs):
+        kwargs.update(
+            {
+                'split_extension': extension,
+                'split_catchall': True,
+                'disable_zonefile': True,
+            }
+        )
        super().__init__(id, directory, *args, **kwargs)
-        self.extension = extension
-
-    def _zone_directory(self, zone):
-        filename = f'{zone.name[:-1]}{self.extension}'
-        return join(self.directory, filename)
-
-    def list_zones(self):
-        n = len(self.extension) - 1
-        for filename in listdir(self.directory):
-            if not filename.endswith(self.extension):
-                continue
-            yield filename[:-n]
-
-    def populate(self, zone, target=False, lenient=False):
-        self.log.debug(
-            'populate: name=%s, target=%s, lenient=%s',
-            zone.name,
-            target,
-            lenient,
+        self.log.warning(
+            '__init__: DEPRECATED use YamlProvider with split_extension, split_catchall, and disable_zonefile instead, will go away in v2.0'
        )
-
-        if target:
-            # When acting as a target we ignore any existing records so that we
-            # create a completely new copy
-            return False
-
-        before = len(zone.records)
-        yaml_filenames = _list_all_yaml_files(self._zone_directory(zone))
-        self.log.info('populate:   found %s YAML files', len(yaml_filenames))
-        for yaml_filename in yaml_filenames:
-            self._populate_from_file(yaml_filename, zone, lenient)
-
-        self.log.info(
-            'populate:   found %s records, exists=False',
-            len(zone.records) - before,
-        )
-        return False
-
-    def _do_apply(self, desired, data):
-        zone_dir = self._zone_directory(desired)
-        if not isdir(zone_dir):
-            makedirs(zone_dir)
-
-        catchall = dict()
-        for record, config in data.items():
-            if record in self.CATCHALL_RECORD_NAMES:
-                catchall[record] = config
-                continue
-            filename = join(zone_dir, f'{record}.yaml')
-            self.log.debug('_apply:   writing filename=%s', filename)
-            with open(filename, 'w') as fh:
-                record_data = {record: config}
-                safe_dump(record_data, fh)
-        if catchall:
-            # Scrub the trailing . to make filenames more sane.
-            dname = desired.name[:-1]
-            filename = join(zone_dir, f'${dname}.yaml')
-            self.log.debug('_apply:   writing catchall filename=%s', filename)
-            with open(filename, 'w') as fh:
-                safe_dump(catchall, fh)
@@ -16,7 +16,8 @@ fi
 echo "## environment & versions ######################################################"
 python --version
 pip --version
-
+echo "## modules: "
+pip freeze
 echo "## clean up ####################################################################"
 find octodns tests -name "*.pyc" -exec rm {} \;
 rm -f *.pyc
@@ -0,0 +1,5 @@
+---
+only-zone-file:
+  type: TXT
+  value: Only included when zone file processing is enabled
+
@@ -2,8 +2,9 @@
 #
 #

-from os import makedirs
-from os.path import basename, dirname, isdir, isfile, join
+from os import makedirs, remove
+from os.path import dirname, isdir, isfile, join
+from shutil import rmtree
 from unittest import TestCase

 from helpers import TemporaryDirectory
@@ -12,16 +13,15 @@ from yaml.constructor import ConstructorError

 from octodns.idna import idna_encode
 from octodns.provider import ProviderException
-from octodns.provider.base import Plan
-from octodns.provider.yaml import (
-    SplitYamlProvider,
-    YamlProvider,
-    _list_all_yaml_files,
-)
+from octodns.provider.yaml import SplitYamlProvider, YamlProvider
 from octodns.record import Create, NsValue, Record, ValuesMixin
 from octodns.zone import SubzoneRecordException, Zone


+def touch(filename):
+    open(filename, 'w').close()
+
+
 class TestYamlProvider(TestCase):
    def test_provider(self):
        source = YamlProvider('test', join(dirname(__file__), 'config'))
@@ -299,6 +299,7 @@ xn--dj-kia8a:
        self.assertTrue(source.supports(DummyType(self)))

    def test_list_zones(self):
+        # test of pre-existing config that lives on disk
        provider = YamlProvider('test', 'tests/config')
        self.assertEqual(
            [
@@ -307,9 +308,159 @@ xn--dj-kia8a:
                'subzone.unit.tests.',
                'unit.tests.',
            ],
-            sorted(provider.list_zones()),
+            list(provider.list_zones()),
        )

+        # some synthetic tests to explicitly exercise the full functionality
+        with TemporaryDirectory() as td:
+            directory = join(td.dirname)
+
+            # noise
+            touch(join(directory, 'README.txt'))
+            # not a zone.name.yaml
+            touch(join(directory, 'production.yaml'))
+            # non-zone directories
+            makedirs(join(directory, 'directory'))
+            makedirs(join(directory, 'never.matches'))
+
+            # basic yaml zone files
+            touch(join(directory, 'unit.test.yaml'))
+            touch(join(directory, 'sub.unit.test.yaml'))
+            touch(join(directory, 'other.tld.yaml'))
+            touch(join(directory, 'both.tld.yaml'))
+
+            # split zones with .
+            makedirs(join(directory, 'split.test.'))
+            makedirs(join(directory, 'sub.split.test.'))
+            makedirs(join(directory, 'other.split.'))
+            makedirs(join(directory, 'both.tld.'))
+
+            # split zones with .tst
+            makedirs(join(directory, 'split-ext.test.tst'))
+            makedirs(join(directory, 'sub.split-ext.test.tst'))
+            makedirs(join(directory, 'other-ext.split.tst'))
+
+            provider = YamlProvider('test', directory)
+
+            # basic, should only find zone files
+            self.assertEqual(
+                ['both.tld.', 'other.tld.', 'sub.unit.test.', 'unit.test.'],
+                list(provider.list_zones()),
+            )
+
+            # include stuff with . AND basic
+            provider.split_extension = '.'
+            self.assertEqual(
+                [
+                    'both.tld.',
+                    'other.split.',
+                    'other.tld.',
+                    'split.test.',
+                    'sub.split.test.',
+                    'sub.unit.test.',
+                    'unit.test.',
+                ],
+                list(provider.list_zones()),
+            )
+
+            # include stuff with .tst AND basic
+            provider.split_extension = '.tst'
+            self.assertEqual(
+                [
+                    'both.tld.',
+                    'other-ext.split.',
+                    'other.tld.',
+                    'split-ext.test.',
+                    'sub.split-ext.test.',
+                    'sub.unit.test.',
+                    'unit.test.',
+                ],
+                list(provider.list_zones()),
+            )
+
+            # only .tst
+            provider.disable_zonefile = True
+            self.assertEqual(
+                ['other-ext.split.', 'split-ext.test.', 'sub.split-ext.test.'],
+                list(provider.list_zones()),
+            )
+
+            # only . (and both zone)
+            provider.split_extension = '.'
+            self.assertEqual(
+                ['both.tld.', 'other.split.', 'split.test.', 'sub.split.test.'],
+                list(provider.list_zones()),
+            )
+
+    def test_split_sources(self):
+        with TemporaryDirectory() as td:
+            directory = join(td.dirname)
+
+            provider = YamlProvider('test', directory, split_extension='.')
+
+            zone = Zone('déjà.vu.', [])
+            zone_utf8 = join(directory, f'{zone.decoded_name}')
+            zone_idna = join(directory, f'{zone.name}')
+
+            filenames = (
+                '*.yaml',
+                '.yaml',
+                'www.yaml',
+                f'${zone.decoded_name}yaml',
+            )
+
+            # create the utf8 zone dir
+            makedirs(zone_utf8)
+            # nothing in it so we should get nothing back
+            self.assertEqual([], list(provider._split_sources(zone)))
+            # create some record files
+            for filename in filenames:
+                touch(join(zone_utf8, filename))
+            # make sure we see them
+            expected = [join(zone_utf8, f) for f in sorted(filenames)]
+            self.assertEqual(expected, sorted(provider._split_sources(zone)))
+
+            # add a idna zone directory
+            makedirs(zone_idna)
+            for filename in filenames:
+                touch(join(zone_idna, filename))
+            with self.assertRaises(ProviderException) as ctx:
+                list(provider._split_sources(zone))
+            msg = str(ctx.exception)
+            self.assertTrue('Both UTF-8' in msg)
+
+            # delete the utf8 version
+            rmtree(zone_utf8)
+            expected = [join(zone_idna, f) for f in sorted(filenames)]
+            self.assertEqual(expected, sorted(provider._split_sources(zone)))
+
+    def test_zone_sources(self):
+        with TemporaryDirectory() as td:
+            directory = join(td.dirname)
+
+            provider = YamlProvider('test', directory)
+
+            zone = Zone('déjà.vu.', [])
+            utf8 = join(directory, f'{zone.decoded_name}yaml')
+            idna = join(directory, f'{zone.name}yaml')
+
+            # create the utf8 version
+            touch(utf8)
+            # make sure that's what we get back
+            self.assertEqual(utf8, provider._zone_sources(zone))
+
+            # create idna version, both exists
+            touch(idna)
+            with self.assertRaises(ProviderException) as ctx:
+                provider._zone_sources(zone)
+            msg = str(ctx.exception)
+            self.assertTrue('Both UTF-8' in msg)
+
+            # delete the utf8 version
+            remove(utf8)
+            # make sure that we get the idna one back
+            self.assertEqual(idna, provider._zone_sources(zone))
+

 class TestSplitYamlProvider(TestCase):
    def test_list_all_yaml_files(self):
@@ -323,40 +474,16 @@ class TestSplitYamlProvider(TestCase):
            # Create some files, some of them with a .yaml extension, all of
            # them empty.
            for emptyfile in all_files:
-                open(join(directory, emptyfile), 'w').close()
+                touch(join(directory, emptyfile))
            # Do the same for some fake directories
            for emptydir in all_dirs:
                makedirs(join(directory, emptydir))

            # This isn't great, but given the variable nature of the temp dir
            # names, it's necessary.
-            d = list(basename(f) for f in _list_all_yaml_files(directory))
+            d = [join(directory, f) for f in yaml_files]
            self.assertEqual(len(yaml_files), len(d))

-    def test_zone_directory(self):
-        source = SplitYamlProvider(
-            'test', join(dirname(__file__), 'config/split'), extension='.tst'
-        )
-
-        zone = Zone('unit.tests.', [])
-
-        self.assertEqual(
-            join(dirname(__file__), 'config/split', 'unit.tests.tst'),
-            source._zone_directory(zone),
-        )
-
-    def test_apply_handles_existing_zone_directory(self):
-        with TemporaryDirectory() as td:
-            provider = SplitYamlProvider(
-                'test', join(td.dirname, 'config'), extension='.tst'
-            )
-            makedirs(join(td.dirname, 'config', 'does.exist.tst'))
-
-            zone = Zone('does.exist.', [])
-            self.assertTrue(isdir(provider._zone_directory(zone)))
-            provider.apply(Plan(None, zone, [], True))
-            self.assertTrue(isdir(provider._zone_directory(zone)))
-
    def test_provider(self):
        source = SplitYamlProvider(
            'test',
@@ -375,9 +502,23 @@ class TestSplitYamlProvider(TestCase):
        # without it we see everything
        source.populate(zone)
        self.assertEqual(20, len(zone.records))
+        self.assertFalse([r for r in zone.records if r.name.startswith('only')])
+
+        # temporarily enable zone file processing too, we should see one extra
+        # record that came from unit.tests.
+        source.disable_zonefile = False
+        zone_both = Zone('unit.tests.', [])
+        source.populate(zone_both)
+        self.assertEqual(21, len(zone_both.records))
+        n = len([r for r in zone_both.records if r.name == 'only-zone-file'])
+        self.assertEqual(1, n)
+        source.disable_zonefile = True

        source.populate(dynamic_zone)
        self.assertEqual(5, len(dynamic_zone.records))
+        self.assertFalse(
+            [r for r in dynamic_zone.records if r.name.startswith('only')]
+        )

        with TemporaryDirectory() as td:
            # Add some subdirs to make sure that it can create them
@@ -579,7 +720,7 @@ class TestSplitYamlProvider(TestCase):
        )
        copy = source.copy()
        self.assertEqual(source.directory, copy.directory)
-        self.assertEqual(source.extension, copy.extension)
+        self.assertEqual(source.split_extension, copy.split_extension)
        self.assertEqual(source.default_ttl, copy.default_ttl)
        self.assertEqual(source.enforce_order, copy.enforce_order)
        self.assertEqual(