From da5b7d0d1e8456dc2860ceabbdcd036847857a22 Mon Sep 17 00:00:00 2001
From: Ross McFarland <rwmcfa1@gmail.com>
Date: Fri, 21 Jul 2023 13:26:29 -0700
Subject: [PATCH] Sketch out an option for dynamic zone config

---
 octodns/manager.py           | 84 ++++++++++++++++++++++++++----------
 octodns/provider/yaml.py     |  6 +++
 tests/config/processors.yaml |  2 +-
 3 files changed, 68 insertions(+), 24 deletions(-)

diff --git a/octodns/manager.py b/octodns/manager.py
index b8fb0ff..58592eb 100644
--- a/octodns/manager.py
+++ b/octodns/manager.py
@@ -466,6 +466,36 @@ class Manager(object):
         # Return the zone as it's the desired state
         return plans, zone
 
+    def _get_sources(self, decoded_zone_name, config, eligible_sources):
+        try:
+            sources = config['sources']
+        except KeyError:
+            raise ManagerException(
+                f'Zone {decoded_zone_name} is missing sources'
+            )
+
+        if eligible_sources and not [
+            s for s in sources if s in eligible_sources
+        ]:
+            return None
+
+        self.log.info('sync:   sources=%s', sources)
+
+        try:
+            # rather than using a list comprehension, we break this loop
+            # out so that the `except` block below can reference the
+            # `source`
+            collected = []
+            for source in sources:
+                collected.append(self.providers[source])
+            sources = collected
+        except KeyError:
+            raise ManagerException(
+                f'Zone {decoded_zone_name}, unknown ' f'source: {source}'
+            )
+
+        return sources
+
     def sync(
         self,
         eligible_zones=[],
@@ -486,6 +516,31 @@ class Manager(object):
         )
 
         zones = self.config['zones']
+
+        for name, config in list(zones.items()):
+            if not name.startswith('*'):
+                continue
+            # we've found a dynamic config element
+
+            # find its sources
+            sources = self._get_sources(name, config, eligible_sources)
+            self.log.info('sync:   dynamic zone=%s, sources=%s', name, sources)
+            for source in sources:
+                for zone_name in source.list_zones():
+                    if zone_name in zones:
+                        self.log.info(
+                            'sync:      zone=%s already in config, ignoring',
+                            zone_name,
+                        )
+                        continue
+                    self.log.info(
+                        'sync:      adding dynamic zone=%s', zone_name
+                    )
+                    zones[zone_name] = config
+
+            # remove the dynamic config element so we don't try and populate it
+            del zones[name]
+
         if eligible_zones:
             zones = IdnaDict({n: zones.get(n) for n in eligible_zones})
 
@@ -532,12 +587,10 @@ class Manager(object):
                 continue
 
             lenient = config.get('lenient', False)
-            try:
-                sources = config['sources']
-            except KeyError:
-                raise ManagerException(
-                    f'Zone {decoded_zone_name} is missing sources'
-                )
+
+            sources = self._get_sources(
+                decoded_zone_name, config, eligible_sources
+            )
 
             try:
                 targets = config['targets']
@@ -548,9 +601,7 @@ class Manager(object):
 
             processors = config.get('processors', [])
 
-            if eligible_sources and not [
-                s for s in sources if s in eligible_sources
-            ]:
+            if not sources:
                 self.log.info('sync:   no eligible sources, skipping')
                 continue
 
@@ -564,7 +615,7 @@ class Manager(object):
                 self.log.info('sync:   no eligible targets, skipping')
                 continue
 
-            self.log.info('sync:   sources=%s -> targets=%s', sources, targets)
+            self.log.info('sync:   targets=%s', targets)
 
             try:
                 collected = []
@@ -577,19 +628,6 @@ class Manager(object):
                     f'processor: {processor}'
                 )
 
-            try:
-                # rather than using a list comprehension, we break this loop
-                # out so that the `except` block below can reference the
-                # `source`
-                collected = []
-                for source in sources:
-                    collected.append(self.providers[source])
-                sources = collected
-            except KeyError:
-                raise ManagerException(
-                    f'Zone {decoded_zone_name}, unknown ' f'source: {source}'
-                )
-
             try:
                 trgs = []
                 for target in targets:
diff --git a/octodns/provider/yaml.py b/octodns/provider/yaml.py
index 9fd8f04..88d9023 100644
--- a/octodns/provider/yaml.py
+++ b/octodns/provider/yaml.py
@@ -190,6 +190,12 @@ class YamlProvider(BaseProvider):
             join(self.directory, f'{zone.name}yaml'),
         )
 
+    def list_zones(self):
+        for filename in listdir(self.directory):
+            if not filename.endswith('.yaml'):
+                continue
+            yield filename[:-4]
+
     def populate(self, zone, target=False, lenient=False):
         self.log.debug(
             'populate: name=%s, target=%s, lenient=%s',
diff --git a/tests/config/processors.yaml b/tests/config/processors.yaml
index 004492c..e697fbd 100644
--- a/tests/config/processors.yaml
+++ b/tests/config/processors.yaml
@@ -41,6 +41,6 @@ zones:
     processors:
     - doesnt-exist
     sources:
-    - in
+    - config
     targets:
     - dump