From efa118c3c836adb4a08a18435f05cfb1f836fd90 Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Wed, 30 May 2018 14:51:59 -0400 Subject: [PATCH] Tweaked webhooks and Redis settings --- docs/configuration/optional-settings.md | 94 ++++++++++--------- docs/data-model/extras.md | 5 +- .../{webhook-backend.md => webhooks.md} | 2 +- netbox/extras/__init__.py | 2 +- netbox/extras/apps.py | 26 +++-- netbox/extras/webhooks.py | 8 +- netbox/netbox/configuration.example.py | 13 ++- netbox/netbox/settings.py | 26 +++-- netbox/netbox/urls.py | 2 +- 9 files changed, 95 insertions(+), 83 deletions(-) rename docs/miscellaneous/{webhook-backend.md => webhooks.md} (97%) diff --git a/docs/configuration/optional-settings.md b/docs/configuration/optional-settings.md index 58672b68c..600927394 100644 --- a/docs/configuration/optional-settings.md +++ b/docs/configuration/optional-settings.md @@ -207,50 +207,6 @@ When determining the primary IP address for a device, IPv6 is preferred over IPv --- -## Redis Connection Settings - -The following settings are defined in the `REDIS` dictionary, much like the regular database settings. - -## DB - -Default: 0 - -When `WEBHOOK_BACKEND_ENABLED` is `True` connect to the redis database with this ID. This is used in conjunction with the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. - ---- - -## DEFAULT_TIMEOUT - -Default: 300 - -When `WEBHOOK_BACKEND_ENABLED` is `True` use this value as the redis timeout. This is used in conjunction with the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. - ---- - -## HOST - -Default: localhost - -When `WEBHOOK_BACKEND_ENABLED` is `True` connect to this redis server host. This is used in conjunction with the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. - ---- - -## PASSWORD - -Default: N/A (empty string value) - -When `WEBHOOK_BACKEND_ENABLED` is `True` use this password to connect to the redis server. This is used in conjunction with the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. - ---- - -## PORT - -Default: 6379 - -When `WEBHOOK_BACKEND_ENABLED` is `True` use this port to connect to the redis server. This is used in conjunction with the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. - ---- - ## REPORTS_ROOT Default: $BASE_DIR/netbox/reports/ @@ -267,11 +223,11 @@ The time zone NetBox will use when dealing with dates and times. It is recommend --- -## WEBHOOK_BACKEND_ENABLED +## WEBHOOKS_ENABLED Default: False -Enable this option to run the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. +Enable this option to run the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhooks/) for more information on setup and use. --- @@ -289,3 +245,49 @@ SHORT_TIME_FORMAT = 'H:i:s' # 13:23:00 DATETIME_FORMAT = 'N j, Y g:i a' # June 26, 2016 1:23 p.m. SHORT_DATETIME_FORMAT = 'Y-m-d H:i' # 2016-06-27 13:23 ``` + +--- + +## Redis Connection Settings + +[Redis](https://redis.io/) is a key-value store which functions as a very lightweight database. It is required when enabling NetBox [webhooks](../miscellaneous/webhooks/). A Redis connection is configured using a dictionary similar to the following: + +``` +REDIS = { + 'HOST': 'localhost', + 'PORT': 6379, + 'PASSWORD': '', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, +} +``` + +### DATABASE + +Default: 0 + +The Redis database ID. + +### DEFAULT_TIMEOUT + +Default: 300 + +The timeout value to use when connecting to the Redis server (in seconds). + +### HOST + +Default: localhost + +The hostname or IP address of the Redis server. + +### PORT + +Default: 6379 + +The TCP port to use when connecting to the Redis server. + +### PASSWORD + +Default: None + +The password to use when authenticating to the Redis server (optional). diff --git a/docs/data-model/extras.md b/docs/data-model/extras.md index a1ea91e57..83f50df65 100644 --- a/docs/data-model/extras.md +++ b/docs/data-model/extras.md @@ -133,7 +133,6 @@ Certain objects within NetBox (namely sites, racks, and devices) can have photos # Webhooks -When the [webhook backend](../miscellaneous/webhook-backend/) is enabled, webhooks define how NetBox should react to events surrounding certain models. The webhook model defines a payload URL and event types to which a set of models should be registered. These event types include `Create`, `Update`, and `Delete`. Upon a matching event, a POST request is sent to the payload URL. An optional `secret` can be configured which will append a `X-Hook-Signature` header to the request, consisting of a HMAC (sha512) hex digest of the request body using the secret as the key. You may also allow a webhook to use insecure ssl. +A webhook defines an HTTP request that is sent to an external application when certain types of objects are created, updated, and/or deleted in NetBox. When a webhook is triggered, a POST request is sent to its configured URL. This request will include a full representation of the object being modified for consumption by the receiver. -!!! warning - Using insecure ssl is generally a bad idea but is allowed as invalid ssl is commonly used in internal IT environments. Using insecure ssl in the webhook means ssl verification when making the POST request will not occur. +An optional secret key can be configured for each webhook. This will append a `X-Hook-Signature` header to the request, consisting of a HMAC (SHA-512) hex digest of the request body using the secret as the key. This digest can be used by the receiver to authenticate the request's content. diff --git a/docs/miscellaneous/webhook-backend.md b/docs/miscellaneous/webhooks.md similarity index 97% rename from docs/miscellaneous/webhook-backend.md rename to docs/miscellaneous/webhooks.md index b1d9b1135..82864f31d 100644 --- a/docs/miscellaneous/webhook-backend.md +++ b/docs/miscellaneous/webhooks.md @@ -132,7 +132,7 @@ Then, restart the supervisor service to detect the changes: act on all netbox processes (netbox-core and netbox-webhook-backend in this case). Now you need only add the configuration settings to connect to redis and enable the webhook backend feature. -- In your `configuration.py` Set [WEBHOOK_BACKEND_ENABLED](../configuration/optional-settings/#webhook_backend_enabled) to `True`. +- In your `configuration.py` Set [WEBHOOKS_ENABLED](../configuration/optional-settings/#webhooks_enabled) to `True`. - If needed, set the optional redis connection settings. By default, they will allow connecting to DB 0 on a locally installed redis server with no password. - [REDIS_DB](../configuration/optional-settings/#redis_db) - [REDIS_DEFAULT_TIMEOUT](../configuration/optional-settings/#redis_default_timeout) diff --git a/netbox/extras/__init__.py b/netbox/extras/__init__.py index 5fdeea52b..c7e9c66ad 100644 --- a/netbox/extras/__init__.py +++ b/netbox/extras/__init__.py @@ -5,7 +5,7 @@ from django.core.exceptions import ImproperlyConfigured default_app_config = 'extras.apps.ExtrasConfig' # check that django-rq is installed and we can connect to redis -if settings.WEBHOOK_BACKEND_ENABLED: +if settings.WEBHOOKS_ENABLED: try: import django_rq except ImportError: diff --git a/netbox/extras/apps.py b/netbox/extras/apps.py index d27962ade..1cac98a23 100644 --- a/netbox/extras/apps.py +++ b/netbox/extras/apps.py @@ -1,8 +1,6 @@ from __future__ import unicode_literals from django.apps import AppConfig -from django.core.cache import caches -from django.db.utils import ProgrammingError from django.core.exceptions import ImproperlyConfigured from django.conf import settings @@ -13,17 +11,25 @@ class ExtrasConfig(AppConfig): def ready(self): import extras.signals - # check that we can connect to redis - if settings.WEBHOOK_BACKEND_ENABLED: + # Check that we can connect to the configured Redis database if webhooks are enabled. + if settings.WEBHOOKS_ENABLED: try: import redis - rs = redis.Redis(settings.REDIS_HOST, - settings.REDIS_PORT, - settings.REDIS_DB, - settings.REDIS_PASSWORD or None) + except ImportError: + raise ImproperlyConfigured( + "WEBHOOKS_ENABLED is True but the redis Python package is not installed. (Try 'pip install " + "redis'.)" + ) + try: + rs = redis.Redis( + host=settings.REDIS_HOST, + port=settings.REDIS_PORT, + db=settings.REDIS_DATABASE, + password=settings.REDIS_PASSWORD or None, + ) rs.ping() except redis.exceptions.ConnectionError: raise ImproperlyConfigured( - "Unable to connect to the redis database. You must provide " - "connection settings to redis per the documentation." + "Unable to connect to the Redis database. Check that the Redis configuration has been defined in " + "configuration.py." ) diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py index 3560bece5..2357e6289 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/webhooks.py @@ -64,7 +64,7 @@ def post_save_receiver(sender, instance, created, **kwargs): Receives post_save signals from registered models. If the webhook backend is enabled, queue any webhooks that apply to the event. """ - if settings.WEBHOOK_BACKEND_ENABLED: + if settings.WEBHOOKS_ENABLED: signal_received_timestamp = time.time() webhook_cache = get_or_set_webhook_cache() # look for any webhooks that match this event @@ -88,7 +88,7 @@ def post_delete_receiver(sender, instance, **kwargs): Receives post_delete signals from registered models. If the webhook backend is enabled, queue any webhooks that apply to the event. """ - if settings.WEBHOOK_BACKEND_ENABLED: + if settings.WEBHOOKS_ENABLED: signal_received_timestamp = time.time() webhook_cache = get_or_set_webhook_cache() obj_type = ContentType.objects.get_for_model(sender) @@ -103,7 +103,7 @@ def bulk_operation_receiver(sender, **kwargs): Receives bulk_operation_signal signals from registered models. If the webhook backend is enabled, queue any webhooks that apply to the event. """ - if settings.WEBHOOK_BACKEND_ENABLED: + if settings.WEBHOOKS_ENABLED: signal_received_timestamp = time.time() event = kwargs['event'] webhook_cache = get_or_set_webhook_cache() @@ -132,7 +132,7 @@ def register_signals(senders): Take a list of senders (Models) and register them to the post_save and post_delete signal receivers. """ - if settings.WEBHOOK_BACKEND_ENABLED: + if settings.WEBHOOKS_ENABLED: # only register signals if the backend is enabled # this reduces load by not firing signals if the # webhook backend feature is disabled diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 1cfcfcf56..27a615c32 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -118,18 +118,17 @@ PAGINATE_COUNT = 50 # prefer IPv4 instead. PREFER_IPV4 = False -# The Webhook event backend is disabled by default. Set this to True to enable it. Besure to follow the documentation -# on first enabling the required components for the webhook backend. -WEBHOOK_BACKEND_ENABLED = False +# The Webhook event backend is disabled by default. Set this to True to enable it. Note that this requires a Redis +# database be configured and accessible by NetBox (see `REDIS` below). +WEBHOOKS_ENABLED = False -# Redis settings. Redis is used in webhook backend so WEBHOOK_BACKEND_ENABLED must be enabled for these -# to mean anything. Please refer to the netbox documentation on the webhook backend. +# Redis database settings (optional). A Redis database is required only if the webhooks backend is enabled. REDIS = { 'HOST': 'localhost', 'PORT': 6379, - 'DEFAULT_TIMEOUT': 300, 'PASSWORD': '', - 'DB': 0, + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, } # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 7aa91092f..44299998e 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -64,13 +64,13 @@ NAPALM_ARGS = getattr(configuration, 'NAPALM_ARGS', {}) PAGINATE_COUNT = getattr(configuration, 'PAGINATE_COUNT', 50) PREFER_IPV4 = getattr(configuration, 'PREFER_IPV4', False) REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/') -WEBHOOK_BACKEND_ENABLED = getattr(configuration, 'WEBHOOK_BACKEND_ENABLED', False) REDIS = getattr(configuration, 'REDIS', {}) SHORT_DATE_FORMAT = getattr(configuration, 'SHORT_DATE_FORMAT', 'Y-m-d') SHORT_DATETIME_FORMAT = getattr(configuration, 'SHORT_DATETIME_FORMAT', 'Y-m-d H:i') SHORT_TIME_FORMAT = getattr(configuration, 'SHORT_TIME_FORMAT', 'H:i:s') TIME_FORMAT = getattr(configuration, 'TIME_FORMAT', 'g:i a') TIME_ZONE = getattr(configuration, 'TIME_ZONE', 'UTC') +WEBHOOKS_ENABLED = getattr(configuration, 'WEBHOOKS_ENABLED', False) CSRF_TRUSTED_ORIGINS = ALLOWED_HOSTS @@ -112,11 +112,17 @@ DATABASES = { } # Redis -REDIS_HOST = REDIS.get('REDIS_HOST', 'localhost') -REDIS_PORT = REDIS.get('REDIS_PORT', 6379) -REDIS_DEFAULT_TIMEOUT = REDIS.get('REDIS_DEFAULT_TIMEOUT', 300) -REDIS_PASSWORD = REDIS.get('REDIS_PASSWORD', '') -REDIS_DB = REDIS.get('REDIS_DB', 0) +REDIS_HOST = REDIS.get('HOST', 'localhost') +REDIS_PORT = REDIS.get('PORT', 6379) +REDIS_PASSWORD = REDIS.get('PASSWORD', '') +REDIS_DATABASE = REDIS.get('DATABASE', 0) +REDIS_DEFAULT_TIMEOUT = REDIS.get('DEFAULT_TIMEOUT', 300) + +print(REDIS_HOST) +print(REDIS_PORT) +print(REDIS_PASSWORD) +print(REDIS_DATABASE) +print(REDIS_DEFAULT_TIMEOUT) # Email EMAIL_HOST = EMAIL.get('SERVER') @@ -156,8 +162,8 @@ INSTALLED_APPS = [ 'drf_yasg', ] -# only load django-rq if the webhook backend is enabled -if WEBHOOK_BACKEND_ENABLED: +# Only load django-rq if the webhook backend is enabled +if WEBHOOKS_ENABLED: INSTALLED_APPS.append('django_rq') # Middleware @@ -259,12 +265,12 @@ REST_FRAMEWORK = { 'VIEW_NAME_FUNCTION': 'netbox.api.get_view_name', } -# Django RQ (Webhook backend) +# Django RQ (Webhooks backend) RQ_QUEUES = { 'default': { 'HOST': REDIS_HOST, 'PORT': REDIS_PORT, - 'DB': REDIS_DB, + 'DB': REDIS_DATABASE, 'PASSWORD': REDIS_PASSWORD, 'DEFAULT_TIMEOUT': REDIS_DEFAULT_TIMEOUT, } diff --git a/netbox/netbox/urls.py b/netbox/netbox/urls.py index 6ba7d14fc..b6f5ca8e8 100644 --- a/netbox/netbox/urls.py +++ b/netbox/netbox/urls.py @@ -64,7 +64,7 @@ _patterns = [ ] -if settings.WEBHOOK_BACKEND_ENABLED: +if settings.WEBHOOKS_ENABLED: _patterns += [ url(r'^admin/webhook-backend-status/', include('django_rq.urls')), ]