Merge pull request #1140 from octodns/secrets

Secrets handlers support

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## v1.6.x - 2024-??-?? - ???
 
 * Add EnsureTrailingDots processor
+* Beta support for custom secret providers added to Manager.
 
 ## v1.5.1 - 2024-03-08 - env/* type conversion fix
 

octodns/manager.py

@@ -10,7 +10,6 @@ from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version as module_version
 from json import dumps
 from logging import getLogger
-from os import environ
 from sys import stdout
 
 from . import __version__
@@ -20,6 +19,7 @@ from .processor.meta import MetaProcessor
 from .provider.base import BaseProvider
 from .provider.plan import Plan
 from .provider.yaml import SplitYamlProvider, YamlProvider
+from .secret.environ import EnvironSecrets
 from .yaml import safe_load
 from .zone import Zone
 
@@ -119,6 +119,14 @@ class Manager(object):
             manager_config, enable_checksum
         )
 
+        # add our hard-coded environ handler first so that other secret
+        # providers can pull in env variables w/it
+        self.secret_handlers = {'env': EnvironSecrets('env')}
+        secret_handlers_config = self.config.get('secret_handlers', {})
+        self.secret_handlers.update(
+            self._config_secret_handlers(secret_handlers_config)
+        )
+
         self.auto_arpa = self._config_auto_arpa(manager_config, auto_arpa)
 
         self.global_processors = manager_config.get('processors', [])
@@ -219,6 +227,38 @@ class Manager(object):
         self.log.info('_config_auto_arpa: auto_arpa=%s', auto_arpa)
         return auto_arpa
 
+    def _config_secret_handlers(self, secret_handlers_config):
+        self.log.debug('_config_secret_handlers: configuring secret_handlers')
+        secret_handlers = {}
+        for sh_name, sh_config in secret_handlers_config.items():
+            # Get our class and remove it from the secret handler config
+            try:
+                _class = sh_config.pop('class')
+            except KeyError:
+                self.log.exception('Invalid secret handler class')
+                raise ManagerException(
+                    f'Secret Handler {sh_name} is missing class, {sh_config.context}'
+                )
+            _class, module, version = self._get_named_class(
+                'secret handler', _class, sh_config.context
+            )
+            kwargs = self._build_kwargs(sh_config)
+            try:
+                secret_handlers[sh_name] = _class(sh_name, **kwargs)
+                self.log.info(
+                    '__init__: secret_handler=%s (%s %s)',
+                    sh_name,
+                    module,
+                    version,
+                )
+            except TypeError:
+                self.log.exception('Invalid secret handler config')
+                raise ManagerException(
+                    f'Incorrect secret handler config for {sh_name}, {sh_config.context}'
+                )
+
+        return secret_handlers
+
     def _config_providers(self, providers_config):
         self.log.debug('_config_providers: configuring providers')
         providers = {}
@@ -362,7 +402,7 @@ class Manager(object):
             return getattr(module, class_name), module_name, version
         except AttributeError:
             self.log.exception(
-                '_get_{}_class: Unable to get class %s from module %s',
+                '_get_named_class: Unable to get class %s from module %s',
                 class_name,
                 module,
             )
@@ -377,26 +417,34 @@ class Manager(object):
             if isinstance(v, dict):
                 v = self._build_kwargs(v)
             elif isinstance(v, str):
-                if v.startswith('env/'):
-                    # expand env variables
+                if '/' in v:
+                    handler, name = v.split('/', 1)
                     try:
-                        env_var = v[4:]
-                        v = environ[env_var]
+                        handler = self.secret_handlers[handler]
                     except KeyError:
-                        self.log.exception('Invalid provider config')
-                        raise ManagerException(
-                            f'Incorrect provider config, missing env var {env_var}, {source.context}'
+                        # we don't have a matching handler, but don't want to
+                        # make that an error b/c config values will often
+                        # contain /. We don't want to print the values in case
+                        # they're sensitive so just provide the key, and even
+                        # that only at debug level.
+                        self.log.debug(
+                            '_build_kwargs: failed to find handler for key "%sp ',
+                            k,
                         )
-                    try:
-                        if '.' in v:
-                            # has a dot, try converting it to a float
-                            v = float(v)
-                        else:
-                            # no dot, try converting it to an int
-                            v = int(v)
-                    except ValueError:
-                        # just leave it as a string
-                        pass
+                    else:
+                        v = handler.fetch(name, source)
+
+                    if isinstance(v, str):
+                        try:
+                            if '.' in v:
+                                # has a dot, try converting it to a float
+                                v = float(v)
+                            else:
+                                # no dot, try converting it to an int
+                                v = int(v)
+                        except ValueError:
+                            # just leave it as a string
+                            pass
 
             kwargs[k] = v
 

octodns/secret/__init__.py

@@ -0,0 +1,3 @@
+#
+#
+#

octodns/secret/base.py

@@ -0,0 +1,11 @@
+#
+#
+#
+
+from logging import getLogger
+
+
+class BaseSecrets:
+    def __init__(self, name):
+        self.log = getLogger(f'{self.__class__.__name__}[{name}]')
+        self.name = name

octodns/secret/environ.py

@@ -0,0 +1,32 @@
+#
+#
+#
+
+from os import environ
+
+from .base import BaseSecrets
+from .exception import SecretsException
+
+
+class EnvironSecretsException(SecretsException):
+    pass
+
+
+class EnvironSecrets(BaseSecrets):
+    def fetch(self, name, source):
+        # expand env variables
+        try:
+            v = environ[name]
+        except KeyError:
+            self.log.exception('Invalid provider config')
+            raise EnvironSecretsException(
+                f'Incorrect provider config, missing env var {name}, {source.context}'
+            )
+        try:
+            # try converting the value to a number to see if it
+            # converts
+            v = float(v)
+        except ValueError:
+            pass
+
+        return v

octodns/secret/exception.py

@@ -0,0 +1,7 @@
+#
+#
+#
+
+
+class SecretsException(Exception):
+    pass

tests/config/secrets.yaml

@@ -0,0 +1,19 @@
+---
+secret_handlers:
+  dummy:
+    class: helpers.DummySecrets
+    prefix: in_config/
+  requires-env:
+    class: helpers.DummySecrets
+    # things can pull from env, it prexists
+    prefix: env/FROM_ENV_WILL_WORK
+  requires-dummy:
+    class: helpers.DummySecrets
+    # things can't pull from other handlers, the order they're configured in is
+    # indeterminent so it's not safe, they're also all added at once
+    prefix: dummy/FROM_DUMMY_WONT_WORK
+
+# Not needed, but required key
+providers: {}
+# Not needed, but required key
+zones: {}

tests/helpers.py

@@ -9,6 +9,7 @@ from tempfile import mkdtemp
 from octodns.processor.base import BaseProcessor
 from octodns.provider.base import BaseProvider
 from octodns.provider.yaml import YamlProvider
+from octodns.secret.base import BaseSecrets
 
 
 class SimpleSource(object):
@@ -134,3 +135,13 @@ class CountingProcessor(BaseProcessor):
     def process_source_zone(self, zone, *args, **kwargs):
         self.count += len(zone.records)
         return zone
+
+
+class DummySecrets(BaseSecrets):
+    def __init__(self, name, prefix):
+        super().__init__(name)
+        self.log.info('__init__: name=%s, prefix=%s', name, prefix)
+        self.prefix = prefix
+
+    def fetch(self, name, source):
+        return f'{self.prefix}{name}'

tests/test_octodns_manager.py

@@ -8,6 +8,7 @@ from unittest import TestCase
 from unittest.mock import MagicMock, patch
 
 from helpers import (
+    DummySecrets,
     DynamicProvider,
     GeoProvider,
     NoSshFpProvider,
@@ -17,6 +18,7 @@ from helpers import (
 )
 
 from octodns import __version__
+from octodns.context import ContextDict
 from octodns.idna import IdnaDict, idna_encode
 from octodns.manager import (
     MainThreadExecutor,
@@ -26,6 +28,7 @@ from octodns.manager import (
 )
 from octodns.processor.base import BaseProcessor
 from octodns.record import Create, Delete, Record, Update
+from octodns.secret.environ import EnvironSecretsException
 from octodns.yaml import safe_load
 from octodns.zone import Zone
 
@@ -68,7 +71,8 @@ class TestManager(TestCase):
         self.assertTrue('provider config' in str(ctx.exception))
 
     def test_missing_env_config(self):
-        with self.assertRaises(ManagerException) as ctx:
+        # details of the EnvironSecrets will be tested in dedicated tests
+        with self.assertRaises(EnvironSecretsException) as ctx:
             Manager(get_config_filename('missing-provider-env.yaml')).sync()
         self.assertTrue('missing env var' in str(ctx.exception))
 
@@ -1215,6 +1219,81 @@ class TestManager(TestCase):
             ),
         )
 
+    def test_config_secret_handlers(self):
+        # config doesn't matter here
+        manager = Manager(get_config_filename('simple.yaml'))
+
+        # no config
+        self.assertEqual({}, manager._config_secret_handlers({}))
+
+        # missing class
+        with self.assertRaises(ManagerException) as ctx:
+            cfg = {'secr3t': ContextDict({}, context='xyz')}
+            manager._config_secret_handlers(cfg)
+        self.assertEqual(
+            'Secret Handler secr3t is missing class, xyz', str(ctx.exception)
+        )
+
+        # bad param
+        with self.assertRaises(ManagerException) as ctx:
+            cfg = {
+                'secr3t': ContextDict(
+                    {
+                        'class': 'octodns.secret.environ.EnvironSecrets',
+                        'bad': 'param',
+                    },
+                    context='xyz',
+                )
+            }
+            manager._config_secret_handlers(cfg)
+        self.assertEqual(
+            'Incorrect secret handler config for secr3t, xyz',
+            str(ctx.exception),
+        )
+
+        # valid with a param that gets used/tested
+        cfg = {
+            'secr3t': ContextDict(
+                {'class': 'helpers.DummySecrets', 'prefix': 'pre-'},
+                context='xyz',
+            )
+        }
+        shs = manager._config_secret_handlers(cfg)
+        sh = shs.get('secr3t')
+        self.assertTrue(sh)
+        self.assertEqual('pre-thing', sh.fetch('thing', None))
+
+        # test configuring secret handlers
+        environ['FROM_ENV_WILL_WORK'] = 'fetched_from_env/'
+        manager = Manager(get_config_filename('secrets.yaml'))
+
+        # dummy was configured
+        self.assertTrue('dummy' in manager.secret_handlers)
+        dummy = manager.secret_handlers['dummy']
+        self.assertIsInstance(dummy, DummySecrets)
+        # and has the prefix value explicitly stated in the yaml
+        self.assertEqual('in_config/hello', dummy.fetch('hello', None))
+
+        # requires-env was configured
+        self.assertTrue('requires-env' in manager.secret_handlers)
+        requires_env = manager.secret_handlers['requires-env']
+        self.assertIsInstance(requires_env, DummySecrets)
+        # and successfully pulled a value from env as its prefix
+        self.assertEqual(
+            'fetched_from_env/hello', requires_env.fetch('hello', None)
+        )
+
+        # requires-dummy was created
+        self.assertTrue('requires-dummy' in manager.secret_handlers)
+        requires_dummy = manager.secret_handlers['requires-dummy']
+        self.assertIsInstance(requires_dummy, DummySecrets)
+        # but failed to fetch a secret from dummy so we just get the configured
+        # value as it was in the yaml for prefix
+        self.assertEqual(
+            'dummy/FROM_DUMMY_WONT_WORK:hello',
+            requires_dummy.fetch(':hello', None),
+        )
+
 
 class TestMainThreadExecutor(TestCase):
     def test_success(self):

tests/test_octodns_secret_environ.py

@@ -0,0 +1,31 @@
+#
+#
+#
+
+from os import environ
+from unittest import TestCase
+
+from octodns.context import ContextDict
+from octodns.secret.environ import EnvironSecrets, EnvironSecretsException
+
+
+class TestEnvironSecrets(TestCase):
+    def test_environ_secrets(self):
+        # put some secrets into our env
+        environ['THIS_EXISTS'] = 'and has a val'
+        environ['THIS_IS_AN_INT'] = '42'
+        environ['THIS_IS_A_FLOAT'] = '43.44'
+
+        es = EnvironSecrets('env')
+
+        source = ContextDict({}, context='xyz')
+        self.assertEqual('and has a val', es.fetch('THIS_EXISTS', source))
+        self.assertEqual(42, es.fetch('THIS_IS_AN_INT', source))
+        self.assertEqual(43.44, es.fetch('THIS_IS_A_FLOAT', source))
+
+        with self.assertRaises(EnvironSecretsException) as ctx:
+            es.fetch('DOES_NOT_EXIST', source)
+        self.assertEqual(
+            'Incorrect provider config, missing env var DOES_NOT_EXIST, xyz',
+            str(ctx.exception),
+        )