From 0d15ac15aeedb71c97e12e325138f1406ff5268f Mon Sep 17 00:00:00 2001 From: John Anderson Date: Sun, 13 Oct 2019 02:49:54 -0400 Subject: [PATCH] implements #3282 - seperate webhooks and caching redis configs --- docs/configuration/required-settings.md | 44 +++++++++++-------- docs/installation/2-netbox.md | 23 ++++++---- docs/release-notes/version-2.7.md | 48 +++++++++++++++++++++ netbox/extras/apps.py | 10 ++--- netbox/netbox/configuration.example.py | 25 +++++++---- netbox/netbox/settings.py | 56 +++++++++++++++++-------- 6 files changed, 152 insertions(+), 54 deletions(-) diff --git a/docs/configuration/required-settings.md b/docs/configuration/required-settings.md index e4f2aed97..f3ea9245b 100644 --- a/docs/configuration/required-settings.md +++ b/docs/configuration/required-settings.md @@ -24,7 +24,7 @@ NetBox requires access to a PostgreSQL database service to store data. This serv Example: -``` +```python DATABASE = { 'NAME': 'netbox', # Database name 'USER': 'netbox', # PostgreSQL username @@ -40,40 +40,48 @@ DATABASE = { [Redis](https://redis.io/) is an in-memory data store similar to memcached. While Redis has been an optional component of NetBox since the introduction of webhooks in version 2.4, it is required starting in 2.6 to support NetBox's caching -functionality (as well as other planned features). +functionality (as well as other planned features). In 2.7, the connection settings were broken down into two sections for +webhooks and caching, allowing the user to connect to different Redis instances/databases per feature. -Redis is configured using a configuration setting similar to `DATABASE`: +Redis is configured using a configuration setting similar to `DATABASE` and these settings are the same for both of the `webhooks` and `caching` subsections: * `HOST` - Name or IP address of the Redis server (use `localhost` if running locally) * `PORT` - TCP port of the Redis service; leave blank for default port (6379) * `PASSWORD` - Redis password (if set) -* `DATABASE` - Numeric database ID for webhooks -* `CACHE_DATABASE` - Numeric database ID for caching +* `DATABASE` - Numeric database ID * `DEFAULT_TIMEOUT` - Connection timeout in seconds * `SSL` - Use SSL connection to Redis Example: -``` +```python REDIS = { - 'HOST': 'localhost', - 'PORT': 6379, - 'PASSWORD': '', - 'DATABASE': 0, - 'CACHE_DATABASE': 1, - 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'webhooks': { + 'HOST': 'redis.example.com', + 'PORT': 1234, + 'PASSWORD': 'foobar', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'HOST': 'localhost', + 'PORT': 6379, + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } } ``` !!! note: - If you were using these settings in a prior release with webhooks, the `DATABASE` setting remains the same but - an additional `CACHE_DATABASE` setting has been added with a default value of 1 to support the caching backend. The - `DATABASE` setting will be renamed in a future release of NetBox to better relay the meaning of the setting. + If you are upgrading from a version prior to v2.7, please note that the Redis connection configuration settings have + changed. Manual modification to bring the `REDIS` section inline with the above specification is necessary !!! warning: - It is highly recommended to keep the webhook and cache databases seperate. Using the same database number for both may result in webhook - processing data being lost in cache flushing events. + It is highly recommended to keep the webhook and cache databases seperate. Using the same database number on the + same Redis instance for both may result in webhook processing data being lost during cache flushing events. --- diff --git a/docs/installation/2-netbox.md b/docs/installation/2-netbox.md index 4add03300..4f11fee83 100644 --- a/docs/installation/2-netbox.md +++ b/docs/installation/2-netbox.md @@ -138,13 +138,22 @@ Redis is a in-memory key-value store required as part of the NetBox installation ```python REDIS = { - 'HOST': 'localhost', - 'PORT': 6379, - 'PASSWORD': '', - 'DATABASE': 0, - 'CACHE_DATABASE': 1, - 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'webhooks': { + 'HOST': 'redis.example.com', + 'PORT': 1234, + 'PASSWORD': 'foobar', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'HOST': 'localhost', + 'PORT': 6379, + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } } ``` diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index d997aa9c2..3504eccdc 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -1,5 +1,53 @@ v2.7.0 (FUTURE) +## Changes + +### Redis Configuration ([#3282](https://github.com/netbox-community/netbox/issues/3282)) + +v2.6.0 introduced caching and added the `CACHE_DATABASE` option to the existing `REDIS` database configuration section. +This did not however, allow for using two different Redis connections for the seperate caching and webhooks features. +This change separates the Redis connection configurations in the `REDIS` section into distinct `webhooks` and `caching` subsections. +This requires modification of the `REDIS` section of the `configuration.py` file as follows: + +Old Redis configuration: +```python +REDIS = { + 'HOST': 'localhost', + 'PORT': 6379, + 'PASSWORD': '', + 'DATABASE': 0, + 'CACHE_DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, +} +``` + +New Redis configuration: +```python +REDIS = { + 'webhooks': { + 'HOST': 'redis.example.com', + 'PORT': 1234, + 'PASSWORD': 'foobar', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'HOST': 'localhost', + 'PORT': 6379, + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } +} +``` + +Note that `CACHE_DATABASE` has been removed and the connection settings have been duplicated for both `webhooks` and `caching`. +This allows the user to make use of separate Redis instances and/or databases if desired. +Full connection details are required in both sections, even if they are the same. + ## Enhancements * [#2902](https://github.com/digitalocean/netbox/issues/2902) - Replace supervisord with systemd diff --git a/netbox/extras/apps.py b/netbox/extras/apps.py index 6bb3b9fca..25c7cd5a2 100644 --- a/netbox/extras/apps.py +++ b/netbox/extras/apps.py @@ -21,11 +21,11 @@ class ExtrasConfig(AppConfig): ) try: rs = redis.Redis( - host=settings.REDIS_HOST, - port=settings.REDIS_PORT, - db=settings.REDIS_DATABASE, - password=settings.REDIS_PASSWORD or None, - ssl=settings.REDIS_SSL, + host=settings.WEBHOOKS_REDIS_HOST, + port=settings.WEBHOOKS_REDIS_PORT, + db=settings.WEBHOOKS_REDIS_DATABASE, + password=settings.WEBHOOKS_REDIS_PASSWORD or None, + ssl=settings.WEBHOOKS_REDIS_SSL, ) rs.ping() except redis.exceptions.ConnectionError: diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index ebc3d4540..d5ac444d2 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -26,14 +26,25 @@ DATABASE = { SECRET_KEY = '' # Redis database settings. The Redis database is used for caching and background processing such as webhooks +# Seperate sections for webhooks and caching allow for connecting to seperate Redis instances/datbases if desired. +# Full connection details are required in both sections, even if they are the same. REDIS = { - 'HOST': 'localhost', - 'PORT': 6379, - 'PASSWORD': '', - 'DATABASE': 0, - 'CACHE_DATABASE': 1, - 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'webhooks': { + 'HOST': 'redis.example.com', + 'PORT': 1234, + 'PASSWORD': 'foobar', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'HOST': 'localhost', + 'PORT': 6379, + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } } diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index c2f51d295..a7c797fce 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -118,13 +118,30 @@ DATABASES = { # Redis # -REDIS_HOST = REDIS.get('HOST', 'localhost') -REDIS_PORT = REDIS.get('PORT', 6379) -REDIS_PASSWORD = REDIS.get('PASSWORD', '') -REDIS_DATABASE = REDIS.get('DATABASE', 0) -REDIS_CACHE_DATABASE = REDIS.get('CACHE_DATABASE', 1) -REDIS_DEFAULT_TIMEOUT = REDIS.get('DEFAULT_TIMEOUT', 300) -REDIS_SSL = REDIS.get('SSL', False) +if 'webhooks' not in REDIS: + raise ImproperlyConfigured( + "REDIS section in configuration.py is missing webhooks subsection." + ) +if 'caching' not in REDIS: + raise ImproperlyConfigured( + "REDIS section in configuration.py is missing caching subsection." + ) + +WEBHOOKS_REDIS = REDIS.get('webhooks', {}) +WEBHOOKS_REDIS_HOST = WEBHOOKS_REDIS.get('HOST', 'localhost') +WEBHOOKS_REDIS_PORT = WEBHOOKS_REDIS.get('PORT', 6379) +WEBHOOKS_REDIS_PASSWORD = WEBHOOKS_REDIS.get('PASSWORD', '') +WEBHOOKS_REDIS_DATABASE = WEBHOOKS_REDIS.get('DATABASE', 0) +WEBHOOKS_REDIS_DEFAULT_TIMEOUT = WEBHOOKS_REDIS.get('DEFAULT_TIMEOUT', 300) +WEBHOOKS_REDIS_SSL = WEBHOOKS_REDIS.get('SSL', False) + +CACHING_REDIS = REDIS.get('caching', {}) +CACHING_REDIS_HOST = WEBHOOKS_REDIS.get('HOST', 'localhost') +CACHING_REDIS_PORT = WEBHOOKS_REDIS.get('PORT', 6379) +CACHING_REDIS_PASSWORD = WEBHOOKS_REDIS.get('PASSWORD', '') +CACHING_REDIS_DATABASE = WEBHOOKS_REDIS.get('DATABASE', 0) +CACHING_REDIS_DEFAULT_TIMEOUT = WEBHOOKS_REDIS.get('DEFAULT_TIMEOUT', 300) +CACHING_REDIS_SSL = WEBHOOKS_REDIS.get('SSL', False) # @@ -341,15 +358,20 @@ if LDAP_CONFIG is not None: # Caching # -if REDIS_SSL: +if CACHING_REDIS_SSL: REDIS_CACHE_CON_STRING = 'rediss://' else: REDIS_CACHE_CON_STRING = 'redis://' -if REDIS_PASSWORD: - REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, REDIS_PASSWORD) +if CACHING_REDIS_PASSWORD: + REDIS_CACHE_CON_STRING = '{}:{}@'.format(REDIS_CACHE_CON_STRING, CACHING_REDIS_PASSWORD) -REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format(REDIS_CACHE_CON_STRING, REDIS_HOST, REDIS_PORT, REDIS_CACHE_DATABASE) +REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format( + REDIS_CACHE_CON_STRING, + CACHING_REDIS_HOST, + CACHING_REDIS_PORT, + CACHING_REDIS_DATABASE +) if not CACHE_TIMEOUT: CACHEOPS_ENABLED = False @@ -467,12 +489,12 @@ SWAGGER_SETTINGS = { RQ_QUEUES = { 'default': { - 'HOST': REDIS_HOST, - 'PORT': REDIS_PORT, - 'DB': REDIS_DATABASE, - 'PASSWORD': REDIS_PASSWORD, - 'DEFAULT_TIMEOUT': REDIS_DEFAULT_TIMEOUT, - 'SSL': REDIS_SSL, + 'HOST': WEBHOOKS_REDIS_HOST, + 'PORT': WEBHOOKS_REDIS_PORT, + 'DB': WEBHOOKS_REDIS_DATABASE, + 'PASSWORD': WEBHOOKS_REDIS_PASSWORD, + 'DEFAULT_TIMEOUT': WEBHOOKS_REDIS_DEFAULT_TIMEOUT, + 'SSL': WEBHOOKS_REDIS_SSL, } }