From 836478c166daa586aef77b88f215f5deab4c09b3 Mon Sep 17 00:00:00 2001 From: John Anderson Date: Wed, 30 May 2018 11:19:10 -0400 Subject: [PATCH] Implements #81 - webhook event backend (#1640) * merge branch develop * bugfix, signals for virtualization's class wasn't correctly defined * updated webhooks for 2.4 and cleanup * updated docs to cover changes to supervisor config * review changes and further cleanup * updated redis connection settings * cleanup settings --- docs/configuration/optional-settings.md | 52 ++++++++ docs/data-model/extras.md | 7 + docs/miscellaneous/webhook-backend.md | 151 ++++++++++++++++++++++ netbox/circuits/apps.py | 5 + netbox/circuits/models.py | 4 + netbox/dcim/apps.py | 6 + netbox/dcim/models.py | 10 ++ netbox/extras/__init__.py | 15 +++ netbox/extras/admin.py | 36 +++++- netbox/extras/api/serializers.py | 1 + netbox/extras/apps.py | 29 +++++ netbox/extras/constants.py | 18 +++ netbox/extras/migrations/0012_webhooks.py | 36 ++++++ netbox/extras/models.py | 74 +++++++++++ netbox/extras/signals.py | 16 +++ netbox/extras/webhooks.py | 142 ++++++++++++++++++++ netbox/extras/webhooks_worker.py | 52 ++++++++ netbox/ipam/apps.py | 7 + netbox/ipam/models.py | 24 ++++ netbox/netbox/configuration.example.py | 14 ++ netbox/netbox/settings.py | 36 +++++- netbox/netbox/urls.py | 6 + netbox/tenancy/apps.py | 7 + netbox/tenancy/models.py | 4 + netbox/utilities/utils.py | 11 ++ netbox/utilities/views.py | 8 ++ netbox/virtualization/__init__.py | 1 + netbox/virtualization/apps.py | 7 + netbox/virtualization/models.py | 6 + 29 files changed, 782 insertions(+), 3 deletions(-) create mode 100644 docs/miscellaneous/webhook-backend.md create mode 100644 netbox/extras/apps.py create mode 100644 netbox/extras/migrations/0012_webhooks.py create mode 100644 netbox/extras/signals.py create mode 100644 netbox/extras/webhooks.py create mode 100644 netbox/extras/webhooks_worker.py diff --git a/docs/configuration/optional-settings.md b/docs/configuration/optional-settings.md index d34137ea0..58672b68c 100644 --- a/docs/configuration/optional-settings.md +++ b/docs/configuration/optional-settings.md @@ -207,6 +207,50 @@ When determining the primary IP address for a device, IPv6 is preferred over IPv --- +## Redis Connection Settings + +The following settings are defined in the `REDIS` dictionary, much like the regular database settings. + +## DB + +Default: 0 + +When `WEBHOOK_BACKEND_ENABLED` is `True` connect to the redis database with this ID. This is used in conjunction with the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. + +--- + +## DEFAULT_TIMEOUT + +Default: 300 + +When `WEBHOOK_BACKEND_ENABLED` is `True` use this value as the redis timeout. This is used in conjunction with the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. + +--- + +## HOST + +Default: localhost + +When `WEBHOOK_BACKEND_ENABLED` is `True` connect to this redis server host. This is used in conjunction with the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. + +--- + +## PASSWORD + +Default: N/A (empty string value) + +When `WEBHOOK_BACKEND_ENABLED` is `True` use this password to connect to the redis server. This is used in conjunction with the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. + +--- + +## PORT + +Default: 6379 + +When `WEBHOOK_BACKEND_ENABLED` is `True` use this port to connect to the redis server. This is used in conjunction with the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. + +--- + ## REPORTS_ROOT Default: $BASE_DIR/netbox/reports/ @@ -223,6 +267,14 @@ The time zone NetBox will use when dealing with dates and times. It is recommend --- +## WEBHOOK_BACKEND_ENABLED + +Default: False + +Enable this option to run the webhook backend. See the docs section on the webhook backend [here](../miscellaneous/webhook-backend/) for more information on setup and use. + +--- + ## Date and Time Formatting You may define custom formatting for date and times. For detailed instructions on writing format strings, please see [the Django documentation](https://docs.djangoproject.com/en/dev/ref/templates/builtins/#date). diff --git a/docs/data-model/extras.md b/docs/data-model/extras.md index f4654c0dd..a1ea91e57 100644 --- a/docs/data-model/extras.md +++ b/docs/data-model/extras.md @@ -130,3 +130,10 @@ Certain objects within NetBox (namely sites, racks, and devices) can have photos !!! note If you experience a server error while attempting to upload an image attachment, verify that the system user NetBox runs as has write permission to the media root directory (`netbox/media/`). + +# Webhooks + +When the [webhook backend](../miscellaneous/webhook-backend/) is enabled, webhooks define how NetBox should react to events surrounding certain models. The webhook model defines a payload URL and event types to which a set of models should be registered. These event types include `Create`, `Update`, and `Delete`. Upon a matching event, a POST request is sent to the payload URL. An optional `secret` can be configured which will append a `X-Hook-Signature` header to the request, consisting of a HMAC (sha512) hex digest of the request body using the secret as the key. You may also allow a webhook to use insecure ssl. + +!!! warning + Using insecure ssl is generally a bad idea but is allowed as invalid ssl is commonly used in internal IT environments. Using insecure ssl in the webhook means ssl verification when making the POST request will not occur. diff --git a/docs/miscellaneous/webhook-backend.md b/docs/miscellaneous/webhook-backend.md new file mode 100644 index 000000000..b1d9b1135 --- /dev/null +++ b/docs/miscellaneous/webhook-backend.md @@ -0,0 +1,151 @@ +# NetBox Webhook Backend + +NetBox includes the ability to send outbound requests to external webhooks upon certain model events occuring, however this functionality is disabled by default and requires some admin interaction to setup. + +When enabled, the user may subscribe webhooks to certain model events. These events include when a model is either created, updated, or deleted. More than one webhook my be registered to a particular model and/or event type. + +## Allowed Models + +The models which may have webhooks registered to them are: + +DCIM: +- Site +- Rack +- RackGroup +- Device +- Interface +IPAM: +- VRF +- IPAddress +- Prefix +- Aggregate +- VLAN +- VLANGroup +- Service +Tenancy: +- Tenant +- TenantGroup +Ciruits: +- Circuit +- Provider +Virtulization: +- Cluster +- ClusterGroup +- VirtualMachine + +## Defining Webhooks + +The [webhook model](../data-model/extras/#webhooks) is used to define a webhook. In general an event type, registered models, and payload url are needed. When a matching event on a registered model occurs, a HTTP POST request is made to the payload url. + +Webhooks are created and updated under extras in the admin site. + +### Request + +The webhook POST request is structured as so (assuming `application/json` as the Content-Type: +``` +{ + "event": "created", + "signal_received_timestamp": 1508769597, + "model": "Site" + "data": { + ... + } +} +``` + +`data` is the serialized representation of the model instance(s) from the event. The same serializers from the NetBox API are used. So an example of the payload for a Site delete event would be: +``` +{ + "event": "deleted", + "signal_received_timestamp": 1508781858.544069, + "model": "Site", + "data": { + "asn": None, + "comments": "", + "contact_email": "", + "contact_name": "", + "contact_phone": "", + "count_circuits": 0, + "count_devices": 0, + "count_prefixes": 0, + "count_racks": 0, + "count_vlans": 0, + "custom_fields": {}, + "facility": "", + "id": 54, + "name": "test", + "physical_address": "", + "region": None, + "shipping_address": "", + "slug": "test", + "tenant": None + } +} +``` + +A request is considered successful if the response status code is any one of a list of "good" statuses defined in the [requests library](https://github.com/requests/requests/blob/205755834d34a8a6ecf2b0b5b2e9c3e6a7f4e4b6/requests/models.py#L688), otherwise the request is marked as having failed. The user may manually retry a failed request. + +## Installation + +The webhook backend feature is considered an "advanced" feature and requires some extra effort to get it running. This is due the fact that a background worker is needed to process events in a non blocking way, i.e. the webhooks are sent in the background as not to interrupt what a user is doing in the NetBox foreground. + +To do this, you must install [Redis](https://redis.io/) or simply be able to connect to an existing redis server. Redis is a lightweight, in memory database. Redis is used as a means of persistance between NetBox and the background worker for the queue of webhooks to be sent. It can be installed through most package managers. +```no-highlight +# apt-get install redis-server +``` + +The only other component needed is [Django-rq](https://github.com/ui/django-rq) which implements [python-rq](http://python-rq.org/) in a native Django context. This should be done from the same place NetBox is installed, i.e. the same python namespace where you run the upgrade script. Python-rq is a simple background job queueing system sitting on top of redis. +```no-highlight +pip install django-rq +``` + +As mentioned before, the feature requires running a background process. This means we need to run another process along side the NetBox application. We can do this conveniently by modifying the supervisord unit used to run NetBox. Taking the configuration provided from the [installation guide](../installation/web-server/#supervisord_installation) modify it to look like this: +```no-highlight +[program:netbox-core] +command = gunicorn -c /opt/netbox/gunicorn_config.py netbox.wsgi +directory = /opt/netbox/netbox/ +user = www-data + +[program:netbox-webhook-backend] +command = python3 /opt/netbox/netbox/manage.py rqworker +directory = /opt/netbox/netbox/ +user = www-data + +[group:netbox] +programs=netbox-core,netbox-webhook-backend +``` +!!! note + `[program:netbox]` was changed to `[program:netbox-core]` + +This allows you to control both the NetBox application and the background worker as one unit. + +Then, restart the supervisor service to detect the changes: + +```no-highlight +# service supervisor restart +``` + +!!! note + Now any time you start or stop netbox using `supervisorctl`, you will need to refer to the + netbox process as `netbox:*` (before this was just `netbox`). This is due to the fact that + we are now running multiple processes with supervisor, and `netbox:*` tells supervisor to + act on all netbox processes (netbox-core and netbox-webhook-backend in this case). + +Now you need only add the configuration settings to connect to redis and enable the webhook backend feature. +- In your `configuration.py` Set [WEBHOOK_BACKEND_ENABLED](../configuration/optional-settings/#webhook_backend_enabled) to `True`. +- If needed, set the optional redis connection settings. By default, they will allow connecting to DB 0 on a locally installed redis server with no password. + - [REDIS_DB](../configuration/optional-settings/#redis_db) + - [REDIS_DEFAULT_TIMEOUT](../configuration/optional-settings/#redis_default_timeout) + - [REDIS_HOST](../configuration/optional-settings/#redis_host) + - [REDIS_PASSWORD](../configuration/optional-settings/#redis_password) + - [REDIS_PORT](../configuration/optional-settings/#redis_port) + +Now you may restart NetBox as normal and the webhook backend should start running! + +```no-highlight +# sudo supervisorctl restart netbox:* +``` + +## Backend Status + +Django-rq includes a status page in the admin site which can be used to view the result of processed webhooks and manually retry any failed webhooks. Access it from http://netbox.local/admin/webhook-backend-status/ diff --git a/netbox/circuits/apps.py b/netbox/circuits/apps.py index 613c347f2..734ba041e 100644 --- a/netbox/circuits/apps.py +++ b/netbox/circuits/apps.py @@ -9,3 +9,8 @@ class CircuitsConfig(AppConfig): def ready(self): import circuits.signals + + # register webhook signals + from extras.webhooks import register_signals + from .models import Circuit, Provider + register_signals([Circuit, Provider]) diff --git a/netbox/circuits/models.py b/netbox/circuits/models.py index cb79b35a4..0a36ba366 100644 --- a/netbox/circuits/models.py +++ b/netbox/circuits/models.py @@ -61,6 +61,8 @@ class Provider(CreatedUpdatedModel, CustomFieldModel): csv_headers = ['name', 'slug', 'asn', 'account', 'portal_url', 'noc_contact', 'admin_contact', 'comments'] + serializer = 'circuits.api.serializers.ProviderSerializer' + class Meta: ordering = ['name'] @@ -175,6 +177,8 @@ class Circuit(CreatedUpdatedModel, CustomFieldModel): 'cid', 'provider', 'type', 'status', 'tenant', 'install_date', 'commit_rate', 'description', 'comments', ] + serializer = 'circuits.api.serializers.CircuitSerializer' + class Meta: ordering = ['provider', 'cid'] unique_together = ['provider', 'cid'] diff --git a/netbox/dcim/apps.py b/netbox/dcim/apps.py index ef3158508..50cdc3a47 100644 --- a/netbox/dcim/apps.py +++ b/netbox/dcim/apps.py @@ -8,4 +8,10 @@ class DCIMConfig(AppConfig): verbose_name = "DCIM" def ready(self): + import dcim.signals + + # register webhook signals + from extras.webhooks import register_signals + from .models import Site, Rack, RackGroup, Device, Interface + register_signals([Site, Rack, Device, Interface, RackGroup]) diff --git a/netbox/dcim/models.py b/netbox/dcim/models.py index 75975437e..19ba4a872 100644 --- a/netbox/dcim/models.py +++ b/netbox/dcim/models.py @@ -169,6 +169,8 @@ class Site(CreatedUpdatedModel, CustomFieldModel): 'shipping_address', 'contact_name', 'contact_phone', 'contact_email', 'comments', ] + serializer = 'dcim.api.serializers.SiteSerializer' + class Meta: ordering = ['name'] @@ -249,6 +251,8 @@ class RackGroup(models.Model): csv_headers = ['site', 'name', 'slug'] + serializer = 'dcim.api.serializers.RackGroupSerializer' + class Meta: ordering = ['site', 'name'] unique_together = [ @@ -397,6 +401,8 @@ class Rack(CreatedUpdatedModel, CustomFieldModel): 'desc_units', 'comments', ] + serializer = 'dcim.api.serializers.RackSerializer' + class Meta: ordering = ['site', 'group', 'name'] unique_together = [ @@ -1243,6 +1249,8 @@ class Device(CreatedUpdatedModel, CustomFieldModel): 'site', 'rack_group', 'rack_name', 'position', 'face', 'comments', ] + serializer = 'dcim.api.serializers.DeviceSerializer' + class Meta: ordering = ['name'] unique_together = [ @@ -1768,6 +1776,8 @@ class Interface(models.Model): objects = InterfaceQuerySet.as_manager() + serializer = 'dcim.api.serializers.InterfaceSerializer' + class Meta: ordering = ['device', 'name'] unique_together = ['device', 'name'] diff --git a/netbox/extras/__init__.py b/netbox/extras/__init__.py index e69de29bb..5fdeea52b 100644 --- a/netbox/extras/__init__.py +++ b/netbox/extras/__init__.py @@ -0,0 +1,15 @@ +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured + + +default_app_config = 'extras.apps.ExtrasConfig' + +# check that django-rq is installed and we can connect to redis +if settings.WEBHOOK_BACKEND_ENABLED: + try: + import django_rq + except ImportError: + raise ImproperlyConfigured( + "django-rq is not installed! You must install this package per " + "the documentation to use the webhook backend." + ) diff --git a/netbox/extras/admin.py b/netbox/extras/admin.py index 07b5a9ae7..e96ae9ac8 100644 --- a/netbox/extras/admin.py +++ b/netbox/extras/admin.py @@ -4,7 +4,11 @@ from django import forms from django.contrib import admin from django.utils.safestring import mark_safe -from .models import CustomField, CustomFieldChoice, Graph, ExportTemplate, TopologyMap, UserAction +from utilities.forms import LaxURLField +from .models import ( + CustomField, CustomFieldChoice, Graph, ExportTemplate, TopologyMap, UserAction, + Webhook +) def order_content_types(field): @@ -15,6 +19,36 @@ def order_content_types(field): field.choices = [(ct.pk, '{} > {}'.format(ct.app_label, ct.name)) for ct in queryset] +# +# Webhooks +# + +class WebhookForm(forms.ModelForm): + + payload_url = LaxURLField() + + class Meta: + model = Webhook + exclude = [] + + def __init__(self, *args, **kwargs): + super(WebhookForm, self).__init__(*args, **kwargs) + + order_content_types(self.fields['obj_type']) + + +@admin.register(Webhook) +class WebhookAdmin(admin.ModelAdmin): + list_display = [ + 'name', 'models', 'payload_url', 'http_content_type', 'enabled', 'type_create', 'type_update', + 'type_delete', 'ssl_verification', + ] + form = WebhookForm + + def models(self, obj): + return ', '.join([ct.name for ct in obj.obj_type.all()]) + + # # Custom fields # diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 5ab28e8b7..6a3c6256f 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -10,6 +10,7 @@ from extras.constants import ACTION_CHOICES, GRAPH_TYPE_CHOICES from extras.models import ExportTemplate, Graph, ImageAttachment, ReportResult, TopologyMap, UserAction from users.api.serializers import NestedUserSerializer from utilities.api import ChoiceFieldSerializer, ContentTypeFieldSerializer, ValidatedModelSerializer +from extras.constants import * # diff --git a/netbox/extras/apps.py b/netbox/extras/apps.py new file mode 100644 index 000000000..d27962ade --- /dev/null +++ b/netbox/extras/apps.py @@ -0,0 +1,29 @@ +from __future__ import unicode_literals + +from django.apps import AppConfig +from django.core.cache import caches +from django.db.utils import ProgrammingError +from django.core.exceptions import ImproperlyConfigured +from django.conf import settings + + +class ExtrasConfig(AppConfig): + name = "extras" + + def ready(self): + import extras.signals + + # check that we can connect to redis + if settings.WEBHOOK_BACKEND_ENABLED: + try: + import redis + rs = redis.Redis(settings.REDIS_HOST, + settings.REDIS_PORT, + settings.REDIS_DB, + settings.REDIS_PASSWORD or None) + rs.ping() + except redis.exceptions.ConnectionError: + raise ImproperlyConfigured( + "Unable to connect to the redis database. You must provide " + "connection settings to redis per the documentation." + ) diff --git a/netbox/extras/constants.py b/netbox/extras/constants.py index 94f58c2d1..8a615c076 100644 --- a/netbox/extras/constants.py +++ b/netbox/extras/constants.py @@ -97,3 +97,21 @@ LOG_LEVEL_CODES = { LOG_WARNING: 'warning', LOG_FAILURE: 'failure', } + +# webhook content types +WEBHOOK_CT_JSON = 1 +WEBHOOK_CT_X_WWW_FORM_ENCODED = 2 +WEBHOOK_CT_CHOICES = ( + (WEBHOOK_CT_JSON, 'application/json'), + (WEBHOOK_CT_X_WWW_FORM_ENCODED, 'application/x-www-form-urlencoded'), +) + +# Models which support registered webhooks +WEBHOOK_MODELS = ( + 'provider', 'circuit', # Circuits + 'site', 'rack', 'rackgroup', 'device', 'interface', # DCIM + 'aggregate', 'prefix', 'ipaddress', 'vlan', 'vlangroup', 'vrf', # IPAM + 'service', + 'tenant', 'tenantgroup', # Tenancy + 'cluster', 'clustergroup', 'virtualmachine', # Virtualization +) diff --git a/netbox/extras/migrations/0012_webhooks.py b/netbox/extras/migrations/0012_webhooks.py new file mode 100644 index 000000000..8ccee6ce8 --- /dev/null +++ b/netbox/extras/migrations/0012_webhooks.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.10 on 2018-05-23 16:35 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('extras', '0011_django2'), + ] + + operations = [ + migrations.CreateModel( + name='Webhook', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=150, unique=True)), + ('type_create', models.BooleanField(default=False, help_text='A POST will be sent to the URL when the object type(s) is created.')), + ('type_update', models.BooleanField(default=False, help_text='A POST will be sent to the URL when the object type(s) is updated.')), + ('type_delete', models.BooleanField(default=False, help_text='A POST will be sent to the URL when the object type(s) is deleted.')), + ('payload_url', models.CharField(max_length=500, verbose_name='A POST will be sent to this URL based on the webhook criteria.')), + ('http_content_type', models.PositiveSmallIntegerField(choices=[(1, 'application/json'), (2, 'application/x-www-form-urlencoded')], default=1)), + ('secret', models.CharField(blank=True, help_text="When provided the request will include a 'X-Hook-Signature' header which is a HMAC hex digest of the payload body using the secret as the key. The secret is not transmitted in the request.", max_length=255)), + ('enabled', models.BooleanField(default=True)), + ('ssl_verification', models.BooleanField(default=True, help_text='By default, use of proper SSL is verified. Disable with caution!')), + ('obj_type', models.ManyToManyField(help_text='The object(s) to which this Webhook applies.', related_name='webhooks', to='contenttypes.ContentType', verbose_name='Object(s)')), + ], + ), + migrations.AlterUniqueTogether( + name='webhook', + unique_together=set([('payload_url', 'type_create', 'type_update', 'type_delete')]), + ), + ] diff --git a/netbox/extras/models.py b/netbox/extras/models.py index 55db7ec25..1a1e13ec5 100644 --- a/netbox/extras/models.py +++ b/netbox/extras/models.py @@ -21,6 +21,80 @@ from utilities.utils import foreground_color from .constants import * +# +# Webhooks +# + +class Webhook(models.Model): + """ + Webhook model that represents all the details for an endoint and how to make a request to + that endpoint with the configured payload. + """ + + obj_type = models.ManyToManyField( + ContentType, + related_name='webhooks', + verbose_name='Object(s)', + limit_choices_to={'model__in': WEBHOOK_MODELS}, + help_text="The object(s) to which this Webhook applies." + ) + name = models.CharField( + max_length=150, + unique=True + ) + type_create = models.BooleanField( + default=False, + help_text="A POST will be sent to the URL when the object type(s) is created." + ) + type_update = models.BooleanField( + default=False, + help_text="A POST will be sent to the URL when the object type(s) is updated." + ) + type_delete = models.BooleanField( + default=False, + help_text="A POST will be sent to the URL when the object type(s) is deleted." + ) + payload_url = models.CharField( + max_length=500, + verbose_name="A POST will be sent to this URL based on the webhook criteria." + ) + http_content_type = models.PositiveSmallIntegerField( + choices=WEBHOOK_CT_CHOICES, + default=WEBHOOK_CT_JSON + ) + secret = models.CharField( + max_length=255, + blank=True, + help_text="When provided the request will include a 'X-Hook-Signature' " + "header which is a HMAC hex digest of the payload body using " + "the secret as the key. The secret is not transmitted in " + "the request." + ) + enabled = models.BooleanField( + default=True + ) + ssl_verification = models.BooleanField( + default=True, + help_text="By default, use of proper SSL is verified. Disable with caution!" + ) + + class Meta: + unique_together = ('payload_url', 'type_create', "type_update", "type_delete",) + + def __str__(self): + return self.name + + def clean(self): + """ + Validate model + """ + + if not self.type_create and not self.type_delete and not self.type_update: + raise ValidationError( + "You must select at least one type. Either create, update, or delete." + ) + + # # Custom fields # diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py new file mode 100644 index 000000000..d676cdf8d --- /dev/null +++ b/netbox/extras/signals.py @@ -0,0 +1,16 @@ +from __future__ import unicode_literals + +from django.db.models.signals import post_delete, post_save +from django.dispatch import receiver +from django.core.cache import caches + +from .models import Webhook + + +@receiver((post_save, post_delete), sender=Webhook) +def update_webhook_cache(**kwargs): + """ + When a Webhook has been modified, update the webhook cache. + """ + cache = caches['default'] + cache.set('webhook_cache', Webhook.objects.all()) diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py new file mode 100644 index 000000000..3560bece5 --- /dev/null +++ b/netbox/extras/webhooks.py @@ -0,0 +1,142 @@ +import time +from importlib import import_module + +from django.db.models.signals import post_save, post_delete +from django.conf import settings +from django.core.cache import caches +from django.dispatch import Signal +from django.contrib.contenttypes.models import ContentType + +from utilities.utils import dynamic_import +from .models import Webhook + + +# +# Webhooks signals regiters and receivers +# + +def get_or_set_webhook_cache(): + """ + Retrieve the webhook cache. If it is None set it to the current + Webhook queryset + """ + cache = caches['default'] + webhook_cache = cache.get('webhook_cache', None) + + if webhook_cache is None: + webhook_cache = Webhook.objects.all() + cache.set('webhook_cache', webhook_cache) + + return webhook_cache + + +def enqueue_webhooks(webhooks, model_class, data, event, signal_received_timestamp): + """ + Serialize data and enqueue webhooks + """ + serializer_context = { + 'request': None, + } + + if isinstance(data, list): + serializer_property = data[0].serializer + serializer_cls = dynamic_import(serializer_property) + serialized_data = serializer_cls(data, context=serializer_context, many=True) + else: + serializer_property = data.serializer + serializer_cls = dynamic_import(serializer_property) + serialized_data = serializer_cls(data, context=serializer_context) + + from django_rq import get_queue + webhook_queue = get_queue('default') + + for webhook in webhooks: + webhook_queue.enqueue("extras.webhooks_worker.process_webhook", + webhook, + serialized_data.data, + model_class, + event, + signal_received_timestamp) + + +def post_save_receiver(sender, instance, created, **kwargs): + """ + Receives post_save signals from registered models. If the webhook + backend is enabled, queue any webhooks that apply to the event. + """ + if settings.WEBHOOK_BACKEND_ENABLED: + signal_received_timestamp = time.time() + webhook_cache = get_or_set_webhook_cache() + # look for any webhooks that match this event + updated = not created + obj_type = ContentType.objects.get_for_model(sender) + webhooks = [ + x + for x in webhook_cache + if ( + x.enabled and x.type_create == created or x.type_update == updated and + obj_type in x.obj_type.all() + ) + ] + event = 'created' if created else 'updated' + if webhooks: + enqueue_webhooks(webhooks, sender, instance, event, signal_received_timestamp) + + +def post_delete_receiver(sender, instance, **kwargs): + """ + Receives post_delete signals from registered models. If the webhook + backend is enabled, queue any webhooks that apply to the event. + """ + if settings.WEBHOOK_BACKEND_ENABLED: + signal_received_timestamp = time.time() + webhook_cache = get_or_set_webhook_cache() + obj_type = ContentType.objects.get_for_model(sender) + # look for any webhooks that match this event + webhooks = [x for x in webhook_cache if x.enabled and x.type_delete and obj_type in x.obj_type.all()] + if webhooks: + enqueue_webhooks(webhooks, sender, instance, 'deleted', signal_received_timestamp) + + +def bulk_operation_receiver(sender, **kwargs): + """ + Receives bulk_operation_signal signals from registered models. If the webhook + backend is enabled, queue any webhooks that apply to the event. + """ + if settings.WEBHOOK_BACKEND_ENABLED: + signal_received_timestamp = time.time() + event = kwargs['event'] + webhook_cache = get_or_set_webhook_cache() + obj_type = ContentType.objects.get_for_model(sender) + # look for any webhooks that match this event + if event == 'created': + webhooks = [x for x in webhook_cache if x.enabled and x.type_create and obj_type in x.obj_type.all()] + elif event == 'updated': + webhooks = [x for x in webhook_cache if x.enabled and x.type_update and obj_type in x.obj_type.all()] + elif event == 'deleted': + webhooks = [x for x in webhook_cache if x.enabled and x.type_delete and obj_type in x.obj_type.all()] + else: + webhooks = None + + if webhooks: + enqueue_webhooks(webhooks, sender, list(kwargs['instances']), event, signal_received_timestamp) + + +# the bulk operation signal is used to overcome signals not being sent for bulk model changes +bulk_operation_signal = Signal(providing_args=["instances", "event"]) +bulk_operation_signal.connect(bulk_operation_receiver) + + +def register_signals(senders): + """ + Take a list of senders (Models) and register them to the post_save + and post_delete signal receivers. + """ + if settings.WEBHOOK_BACKEND_ENABLED: + # only register signals if the backend is enabled + # this reduces load by not firing signals if the + # webhook backend feature is disabled + + for sender in senders: + post_save.connect(post_save_receiver, sender=sender) + post_delete.connect(post_delete_receiver, sender=sender) diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py new file mode 100644 index 000000000..6d346a51f --- /dev/null +++ b/netbox/extras/webhooks_worker.py @@ -0,0 +1,52 @@ +import requests +import hmac +import hashlib +from rq.utils import import_attribute + +from django_rq import job + +from extras.constants import WEBHOOK_CT_JSON, WEBHOOK_CT_X_WWW_FORM_ENCODED + + +@job('default') +def process_webhook(webhook, data, model_class, event, timestamp): + """ + Make a POST request to the defined Webhook + """ + payload = { + 'event': event, + 'timestamp': timestamp, + 'model': model_class.__name__, + 'data': data + } + headers = { + 'Content-Type': webhook.get_http_content_type_display(), + } + params = { + 'method': 'POST', + 'url': webhook.payload_url, + 'headers': headers + } + + if webhook.http_content_type == WEBHOOK_CT_JSON: + params.update({'json': payload}) + elif webhook.http_content_type == WEBHOOK_CT_X_WWW_FORM_ENCODED: + params.update({'data': payload}) + + prepared_request = requests.Request(**params).prepare() + + if webhook.secret != '': + # sign the request with the secret + hmac_prep = hmac.new(bytearray(webhook.secret, 'utf8'), prepared_request.body, digestmod=hashlib.sha512) + prepared_request.headers['X-Hook-Signature'] = hmac_prep.hexdigest() + + with requests.Session() as session: + session.verify = webhook.ssl_verification + response = session.send(prepared_request) + + if response.status_code >= 200 and response.status_code <= 299: + return 'Status {} returned, webhook successfully processed.'.format(response.status_code) + else: + raise requests.exceptions.RequestException( + "Status {} returned, webhook FAILED to process.".format(response.status_code) + ) diff --git a/netbox/ipam/apps.py b/netbox/ipam/apps.py index c944d1b2c..0bde9853a 100644 --- a/netbox/ipam/apps.py +++ b/netbox/ipam/apps.py @@ -6,3 +6,10 @@ from django.apps import AppConfig class IPAMConfig(AppConfig): name = "ipam" verbose_name = "IPAM" + + def ready(self): + + # register webhook signals + from extras.webhooks import register_signals + from .models import Aggregate, Prefix, IPAddress, VLAN, VRF, VLANGroup, Service + register_signals([Aggregate, Prefix, IPAddress, VLAN, VRF, VLANGroup, Service]) diff --git a/netbox/ipam/models.py b/netbox/ipam/models.py index 65a9cce55..fe32baaf5 100644 --- a/netbox/ipam/models.py +++ b/netbox/ipam/models.py @@ -61,6 +61,8 @@ class VRF(CreatedUpdatedModel, CustomFieldModel): csv_headers = ['name', 'rd', 'tenant', 'enforce_unique', 'description'] + serializer = 'ipam.api.serializers.VRFSerializer' + class Meta: ordering = ['name', 'rd'] verbose_name = 'VRF' @@ -162,6 +164,8 @@ class Aggregate(CreatedUpdatedModel, CustomFieldModel): csv_headers = ['prefix', 'rir', 'date_added', 'description'] + serializer = 'ipam.api.serializers.AggregateSerializer' + class Meta: ordering = ['family', 'prefix'] @@ -336,6 +340,8 @@ class Prefix(CreatedUpdatedModel, CustomFieldModel): 'prefix', 'vrf', 'tenant', 'site', 'vlan_group', 'vlan_vid', 'status', 'role', 'is_pool', 'description', ] + serializer = 'ipam.api.serializers.PrefixSerializer' + class Meta: ordering = ['vrf', 'family', 'prefix'] verbose_name_plural = 'prefixes' @@ -481,6 +487,16 @@ class Prefix(CreatedUpdatedModel, CustomFieldModel): prefix_size -= 2 return int(float(child_count) / prefix_size * 100) + def new_subnet(self): + if self.family == 4: + if self.prefix.prefixlen <= 30: + return netaddr.IPNetwork('{}/{}'.format(self.prefix.network, self.prefix.prefixlen + 1)) + return None + if self.family == 6: + if self.prefix.prefixlen <= 126: + return netaddr.IPNetwork('{}/{}'.format(self.prefix.network, self.prefix.prefixlen + 1)) + return None + class IPAddressManager(models.Manager): @@ -577,6 +593,8 @@ class IPAddress(CreatedUpdatedModel, CustomFieldModel): 'description', ] + serializer = 'ipam.api.serializers.IPAddressSerializer' + class Meta: ordering = ['family', 'address'] verbose_name = 'IP address' @@ -673,6 +691,8 @@ class VLANGroup(models.Model): csv_headers = ['name', 'slug', 'site'] + serializer = 'ipam.api.serializers.VLANGroupSerializer' + class Meta: ordering = ['site', 'name'] unique_together = [ @@ -770,6 +790,8 @@ class VLAN(CreatedUpdatedModel, CustomFieldModel): csv_headers = ['site', 'group_name', 'vid', 'name', 'tenant', 'status', 'role', 'description'] + serializer = 'ipam.api.serializers.VLANSerializer' + class Meta: ordering = ['site', 'group', 'vid'] unique_together = [ @@ -864,6 +886,8 @@ class Service(CreatedUpdatedModel): blank=True ) + serializer = 'ipam.api.serializers.ServiceSerializer' + class Meta: ordering = ['protocol', 'port'] diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index 4c4cf4277..1cfcfcf56 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -118,6 +118,20 @@ PAGINATE_COUNT = 50 # prefer IPv4 instead. PREFER_IPV4 = False +# The Webhook event backend is disabled by default. Set this to True to enable it. Besure to follow the documentation +# on first enabling the required components for the webhook backend. +WEBHOOK_BACKEND_ENABLED = False + +# Redis settings. Redis is used in webhook backend so WEBHOOK_BACKEND_ENABLED must be enabled for these +# to mean anything. Please refer to the netbox documentation on the webhook backend. +REDIS = { + 'HOST': 'localhost', + 'PORT': 6379, + 'DEFAULT_TIMEOUT': 300, + 'PASSWORD': '', + 'DB': 0, +} + # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. # REPORTS_ROOT = '/opt/netbox/netbox/reports' diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index ad313a593..7aa91092f 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -64,6 +64,8 @@ NAPALM_ARGS = getattr(configuration, 'NAPALM_ARGS', {}) PAGINATE_COUNT = getattr(configuration, 'PAGINATE_COUNT', 50) PREFER_IPV4 = getattr(configuration, 'PREFER_IPV4', False) REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/') +WEBHOOK_BACKEND_ENABLED = getattr(configuration, 'WEBHOOK_BACKEND_ENABLED', False) +REDIS = getattr(configuration, 'REDIS', {}) SHORT_DATE_FORMAT = getattr(configuration, 'SHORT_DATE_FORMAT', 'Y-m-d') SHORT_DATETIME_FORMAT = getattr(configuration, 'SHORT_DATETIME_FORMAT', 'Y-m-d H:i') SHORT_TIME_FORMAT = getattr(configuration, 'SHORT_TIME_FORMAT', 'H:i:s') @@ -109,6 +111,13 @@ DATABASES = { 'default': configuration.DATABASE, } +# Redis +REDIS_HOST = REDIS.get('REDIS_HOST', 'localhost') +REDIS_PORT = REDIS.get('REDIS_PORT', 6379) +REDIS_DEFAULT_TIMEOUT = REDIS.get('REDIS_DEFAULT_TIMEOUT', 300) +REDIS_PASSWORD = REDIS.get('REDIS_PASSWORD', '') +REDIS_DB = REDIS.get('REDIS_DB', 0) + # Email EMAIL_HOST = EMAIL.get('SERVER') EMAIL_PORT = EMAIL.get('PORT', 25) @@ -119,7 +128,7 @@ SERVER_EMAIL = EMAIL.get('FROM_EMAIL') EMAIL_SUBJECT_PREFIX = '[NetBox] ' # Installed applications -INSTALLED_APPS = ( +INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', @@ -145,7 +154,11 @@ INSTALLED_APPS = ( 'utilities', 'virtualization', 'drf_yasg', -) +] + +# only load django-rq if the webhook backend is enabled +if WEBHOOK_BACKEND_ENABLED: + INSTALLED_APPS.append('django_rq') # Middleware MIDDLEWARE = ( @@ -246,6 +259,17 @@ REST_FRAMEWORK = { 'VIEW_NAME_FUNCTION': 'netbox.api.get_view_name', } +# Django RQ (Webhook backend) +RQ_QUEUES = { + 'default': { + 'HOST': REDIS_HOST, + 'PORT': REDIS_PORT, + 'DB': REDIS_DB, + 'PASSWORD': REDIS_PASSWORD, + 'DEFAULT_TIMEOUT': REDIS_DEFAULT_TIMEOUT, + } +} + # drf_yasg settings for Swagger SWAGGER_SETTINGS = { 'DEFAULT_FIELD_INSPECTORS': [ @@ -278,6 +302,14 @@ INTERNAL_IPS = ( '::1', ) +# Django CACHE - local memory cache +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'LOCATION': 'webhooks', + } +} + try: HOSTNAME = socket.gethostname() diff --git a/netbox/netbox/urls.py b/netbox/netbox/urls.py index 5f7b26a71..6ba7d14fc 100644 --- a/netbox/netbox/urls.py +++ b/netbox/netbox/urls.py @@ -64,6 +64,12 @@ _patterns = [ ] +if settings.WEBHOOK_BACKEND_ENABLED: + _patterns += [ + url(r'^admin/webhook-backend-status/', include('django_rq.urls')), + ] + + if settings.DEBUG: import debug_toolbar _patterns += [ diff --git a/netbox/tenancy/apps.py b/netbox/tenancy/apps.py index df2cd2fbb..6e29d4468 100644 --- a/netbox/tenancy/apps.py +++ b/netbox/tenancy/apps.py @@ -5,3 +5,10 @@ from django.apps import AppConfig class TenancyConfig(AppConfig): name = 'tenancy' + + def ready(self): + + # register webhook signals + from extras.webhooks import register_signals + from .models import Tenant, TenantGroup + register_signals([Tenant, TenantGroup]) diff --git a/netbox/tenancy/models.py b/netbox/tenancy/models.py index f006e512d..bc87ccd8c 100644 --- a/netbox/tenancy/models.py +++ b/netbox/tenancy/models.py @@ -25,6 +25,8 @@ class TenantGroup(models.Model): csv_headers = ['name', 'slug'] + serializer = 'tenancy.api.serializers.TenantGroupSerializer' + class Meta: ordering = ['name'] @@ -79,6 +81,8 @@ class Tenant(CreatedUpdatedModel, CustomFieldModel): csv_headers = ['name', 'slug', 'group', 'description', 'comments'] + serializer = 'tenancy.api.serializers.TenantSerializer' + class Meta: ordering = ['group', 'name'] diff --git a/netbox/utilities/utils.py b/netbox/utilities/utils.py index 9e96a66fd..e995c5580 100644 --- a/netbox/utilities/utils.py +++ b/netbox/utilities/utils.py @@ -71,3 +71,14 @@ def foreground_color(bg_color): return '000000' else: return 'ffffff' + + +def dynamic_import(name): + """ + Dynamically import a class from an absolute path string + """ + components = name.split('.') + mod = __import__(components[0]) + for comp in components[1:]: + mod = getattr(mod, comp) + return mod diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index fd085a1e7..4f70e6215 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -20,6 +20,7 @@ from django.views.generic import View from django_tables2 import RequestConfig from extras.models import CustomField, CustomFieldValue, ExportTemplate, UserAction +from extras.webhooks import bulk_operation_signal from utilities.utils import queryset_to_csv from utilities.forms import BootstrapMixin, CSVDataField from .constants import M2M_FIELD_TYPES @@ -772,6 +773,9 @@ class ComponentCreateView(View): field_links.append(field_link) getattr(self.model, field).through.objects.bulk_create(field_links) + # send the bulk operations signal for webhooks + bulk_operation_signal.send(sender=self.model, instances=new_components, event="created") + messages.success(request, "Added {} {} to {}.".format( len(new_components), self.model._meta.verbose_name_plural, parent )) @@ -848,6 +852,10 @@ class BulkComponentCreateView(View): if not form.errors: self.model.objects.bulk_create(new_components) + + # send the bulk operations signal for webhooks + bulk_operation_signal.send(sender=self.model, instances=new_components, event="created") + messages.success(request, "Added {} {} to {} {}.".format( len(new_components), self.model._meta.verbose_name_plural, diff --git a/netbox/virtualization/__init__.py b/netbox/virtualization/__init__.py index e69de29bb..3f12ae450 100644 --- a/netbox/virtualization/__init__.py +++ b/netbox/virtualization/__init__.py @@ -0,0 +1 @@ +default_app_config = 'virtualization.apps.VirtualizationConfig' diff --git a/netbox/virtualization/apps.py b/netbox/virtualization/apps.py index 768508cfb..359d4bede 100644 --- a/netbox/virtualization/apps.py +++ b/netbox/virtualization/apps.py @@ -5,3 +5,10 @@ from django.apps import AppConfig class VirtualizationConfig(AppConfig): name = 'virtualization' + + def ready(self): + + # register webhook signals + from extras.webhooks import register_signals + from .models import Cluster, ClusterGroup, VirtualMachine + register_signals([Cluster, VirtualMachine, ClusterGroup]) diff --git a/netbox/virtualization/models.py b/netbox/virtualization/models.py index e34512410..42b6591f4 100644 --- a/netbox/virtualization/models.py +++ b/netbox/virtualization/models.py @@ -68,6 +68,8 @@ class ClusterGroup(models.Model): csv_headers = ['name', 'slug'] + serializer = 'virtualization.api.serializers.ClusterGroupSerializer' + class Meta: ordering = ['name'] @@ -129,6 +131,8 @@ class Cluster(CreatedUpdatedModel, CustomFieldModel): csv_headers = ['name', 'type', 'group', 'site', 'comments'] + serializer = 'virtualization.api.serializers.ClusterSerializer' + class Meta: ordering = ['name'] @@ -251,6 +255,8 @@ class VirtualMachine(CreatedUpdatedModel, CustomFieldModel): 'name', 'status', 'role', 'cluster', 'tenant', 'platform', 'vcpus', 'memory', 'disk', 'comments', ] + serializer = 'virtualization.api.serializers.VirtualMachineSerializer' + class Meta: ordering = ['name']