Webhook signal refactor - fixes #2282 (#2260)

Refactor of webhook signaling system to use the same middleware mechanics of Changelogging

netbox/circuits/apps.py

@@ -9,8 +9,3 @@ class CircuitsConfig(AppConfig):
 
     def ready(self):
         import circuits.signals
-
-        # register webhook signals
-        from extras.webhooks import register_signals
-        from .models import Circuit, Provider
-        register_signals([Circuit, Provider])

netbox/circuits/models.py

@@ -60,7 +60,6 @@ class Provider(ChangeLoggedModel, CustomFieldModel):
 
     tags = TaggableManager()
 
-    serializer = 'circuits.api.serializers.ProviderSerializer'
     csv_headers = ['name', 'slug', 'asn', 'account', 'portal_url', 'noc_contact', 'admin_contact', 'comments']
 
     class Meta:
@@ -99,7 +98,6 @@ class CircuitType(ChangeLoggedModel):
         unique=True
     )
 
-    serializer = 'circuits.api.serializers.CircuitTypeSerializer'
     csv_headers = ['name', 'slug']
 
     class Meta:
@@ -174,7 +172,6 @@ class Circuit(ChangeLoggedModel, CustomFieldModel):
 
     tags = TaggableManager()
 
-    serializer = 'circuits.api.serializers.CircuitSerializer'
     csv_headers = [
         'cid', 'provider', 'type', 'status', 'tenant', 'install_date', 'commit_rate', 'description', 'comments',
     ]

netbox/dcim/apps.py

@@ -10,8 +10,3 @@ class DCIMConfig(AppConfig):
     def ready(self):
 
         import dcim.signals
-
-        # register webhook signals
-        from extras.webhooks import register_signals
-        from .models import Site, Rack, RackGroup, Device, Interface
-        register_signals([Site, Rack, Device, Interface, RackGroup])

netbox/dcim/models.py

@@ -79,7 +79,6 @@ class Region(MPTTModel, ChangeLoggedModel):
         unique=True
     )
 
-    serializer = 'dcim.api.serializers.RegionSerializer'
     csv_headers = ['name', 'slug', 'parent']
 
     class MPTTMeta:
@@ -201,7 +200,6 @@ class Site(ChangeLoggedModel, CustomFieldModel):
     objects = SiteManager()
     tags = TaggableManager()
 
-    serializer = 'dcim.api.serializers.SiteSerializer'
     csv_headers = [
         'name', 'slug', 'status', 'region', 'tenant', 'facility', 'asn', 'time_zone', 'description', 'physical_address',
         'shipping_address', 'latitude', 'longitude', 'contact_name', 'contact_phone', 'contact_email', 'comments',
@@ -287,7 +285,6 @@ class RackGroup(ChangeLoggedModel):
         related_name='rack_groups'
     )
 
-    serializer = 'dcim.api.serializers.RackGroupSerializer'
     csv_headers = ['site', 'name', 'slug']
 
     class Meta:
@@ -325,7 +322,6 @@ class RackRole(ChangeLoggedModel):
     )
     color = ColorField()
 
-    serializer = 'dcim.api.serializers.RackRoleSerializer'
     csv_headers = ['name', 'slug', 'color']
 
     class Meta:
@@ -432,7 +428,6 @@ class Rack(ChangeLoggedModel, CustomFieldModel):
     objects = RackManager()
     tags = TaggableManager()
 
-    serializer = 'dcim.api.serializers.RackSerializer'
     csv_headers = [
         'site', 'group_name', 'name', 'facility_id', 'tenant', 'role', 'type', 'serial', 'width', 'u_height',
         'desc_units', 'comments',
@@ -636,8 +631,6 @@ class RackReservation(ChangeLoggedModel):
         max_length=100
     )
 
-    serializer = 'dcim.api.serializers.RackReservationSerializer'
-
     class Meta:
         ordering = ['created']
 
@@ -697,7 +690,6 @@ class Manufacturer(ChangeLoggedModel):
         unique=True
     )
 
-    serializer = 'dcim.api.serializers.ManufacturerSerializer'
     csv_headers = ['name', 'slug']
 
     class Meta:
@@ -792,7 +784,6 @@ class DeviceType(ChangeLoggedModel, CustomFieldModel):
 
     tags = TaggableManager()
 
-    serializer = 'dcim.api.serializers.DeviceTypeSerializer'
     csv_headers = [
         'manufacturer', 'model', 'slug', 'part_number', 'u_height', 'is_full_depth', 'is_console_server',
         'is_pdu', 'is_network_device', 'subdevice_role', 'interface_ordering', 'comments',
@@ -1076,7 +1067,6 @@ class DeviceRole(ChangeLoggedModel):
         help_text='Virtual machines may be assigned to this role'
     )
 
-    serializer = 'dcim.api.serializers.DeviceRoleSerializer'
     csv_headers = ['name', 'slug', 'color', 'vm_role']
 
     class Meta:
@@ -1135,7 +1125,6 @@ class Platform(ChangeLoggedModel):
         verbose_name='Legacy RPC client'
     )
 
-    serializer = 'dcim.api.serializers.PlatformSerializer'
     csv_headers = ['name', 'slug', 'manufacturer', 'napalm_driver', 'napalm_args']
 
     class Meta:
@@ -1302,7 +1291,6 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel):
     objects = DeviceManager()
     tags = TaggableManager()
 
-    serializer = 'dcim.api.serializers.DeviceSerializer'
     csv_headers = [
         'name', 'device_role', 'tenant', 'manufacturer', 'model_name', 'platform', 'serial', 'asset_tag', 'status',
         'site', 'rack_group', 'rack_name', 'position', 'face', 'comments',
@@ -1858,8 +1846,6 @@ class Interface(ComponentModel):
     objects = InterfaceQuerySet.as_manager()
     tags = TaggableManager()
 
-    serializer = 'dcim.api.serializers.InterfaceSerializer'
-
     class Meta:
         ordering = ['device', 'name']
         unique_together = ['device', 'name']
@@ -2263,7 +2249,6 @@ class VirtualChassis(ChangeLoggedModel):
 
     tags = TaggableManager()
 
-    serializer = 'dcim.api.serializers.VirtualChassisSerializer'
     csv_headers = ['master', 'domain']
 
     class Meta:

netbox/extras/middleware.py

@@ -9,7 +9,11 @@ from django.conf import settings
 from django.db.models.signals import post_delete, post_save
 from django.utils import timezone
 
-from .constants import OBJECTCHANGE_ACTION_CREATE, OBJECTCHANGE_ACTION_DELETE, OBJECTCHANGE_ACTION_UPDATE
+from extras.webhooks import enqueue_webhooks
+from .constants import (
+    OBJECTCHANGE_ACTION_CREATE, OBJECTCHANGE_ACTION_DELETE, OBJECTCHANGE_ACTION_UPDATE,
+    WEBHOOK_MODELS
+)
 from .models import ObjectChange
 
 
@@ -18,12 +22,10 @@ _thread_locals = threading.local()
 
 def mark_object_changed(instance, **kwargs):
     """
-    Mark an object as having been created, saved, or updated. At the end of the request, this change will be recorded.
+    Mark an object as having been created, saved, or updated. At the end of the request, this change will be recorded
-    We have to wait until the *end* of the request to the serialize the object, because related fields like tags and
+    and/or associated webhooks fired. We have to wait until the *end* of the request to the serialize the object,
-    custom fields have not yet been updated when the post_save signal is emitted.
+    because related fields like tags and custom fields have not yet been updated when the post_save signal is emitted.
     """
-    if not hasattr(instance, 'log_change'):
-        return
 
     # Determine what action is being performed. The post_save signal sends a `created` boolean, whereas post_delete
     # does not.
@@ -35,7 +37,12 @@ def mark_object_changed(instance, **kwargs):
     _thread_locals.changed_objects.append((instance, action))
 
 
-class ChangeLoggingMiddleware(object):
+class ObjectChangeMiddleware(object):
+    """
+    This middleware intercepts all requests to connects object signals to the Django runtime. The signals collect all
+    changed objects into a local thread by way of the `mark_object_changed()` receiver. At the end of the request,
+    the middleware iterates over the objects to process change events like Change Logging and Webhooks.
+    """
 
     def __init__(self, get_response):
         self.get_response = get_response
@@ -56,11 +63,16 @@ class ChangeLoggingMiddleware(object):
         # Process the request
         response = self.get_response(request)
 
-        # Record object changes
+        # Perform change logging and fire Webhook signals
         for obj, action in _thread_locals.changed_objects:
-            if obj.pk:
+            # Log object changes
+            if obj.pk and hasattr(obj, 'log_change'):
                 obj.log_change(request.user, request.id, action)
 
+            # Enqueue Webhooks if they are enabled
+            if settings.WEBHOOKS_ENABLED and obj.__class__.__name__.lower() in WEBHOOK_MODELS:
+                enqueue_webhooks(obj, action)
+
         # Housekeeping: 1% chance of clearing out expired ObjectChanges
         if _thread_locals.changed_objects and settings.CHANGELOG_RETENTION and random.randint(1, 100) == 1:
             cutoff = timezone.now() - timedelta(days=settings.CHANGELOG_RETENTION)

netbox/extras/models.py

@@ -813,7 +813,6 @@ class ObjectChange(models.Model):
         editable=False
     )
 
-    serializer = 'extras.api.serializers.ObjectChangeSerializer'
     csv_headers = [
         'time', 'user', 'user_name', 'request_id', 'action', 'changed_object_type', 'changed_object_id',
         'related_object_type', 'related_object_id', 'object_repr', 'object_data',

netbox/extras/webhooks.py

@@ -1,119 +1,54 @@
-import time
+import datetime
 
-from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
 from django.db.models import Q
-from django.db.models.signals import post_save, post_delete
-from django.dispatch import Signal
 
 from extras.models import Webhook
-from utilities.utils import dynamic_import
+from extras.constants import OBJECTCHANGE_ACTION_CREATE, OBJECTCHANGE_ACTION_DELETE, OBJECTCHANGE_ACTION_UPDATE
+from utilities.api import get_serializer_for_model
 
 
-def enqueue_webhooks(webhooks, model_class, data, event, signal_received_timestamp):
+def enqueue_webhooks(instance, action):
     """
-    Serialize data and enqueue webhooks
+    Find Webhook(s) assigned to this instance + action and enqueue them
+    to be processed
     """
-    serializer_context = {
+    type_create = action == OBJECTCHANGE_ACTION_CREATE
-        'request': None,
+    type_update = action == OBJECTCHANGE_ACTION_UPDATE
-    }
+    type_delete = action == OBJECTCHANGE_ACTION_DELETE
 
-    if isinstance(data, list):
+    # Find assigned webhooks
-        serializer_property = data[0].serializer
+    obj_type = ContentType.objects.get_for_model(instance.__class__)
-        serializer_cls = dynamic_import(serializer_property)
+    webhooks = Webhook.objects.filter(
-        serialized_data = serializer_cls(data, context=serializer_context, many=True)
+        Q(enabled=True) &
-    else:
+        (
-        serializer_property = data.serializer
+            Q(type_create=type_create) |
-        serializer_cls = dynamic_import(serializer_property)
+            Q(type_update=type_update) |
-        serialized_data = serializer_cls(data, context=serializer_context)
+            Q(type_delete=type_delete)
+        ) &
+        Q(obj_type=obj_type)
+    )
 
-    from django_rq import get_queue
+    if webhooks:
-    webhook_queue = get_queue('default')
+        # Get the Model's API serializer class and serialize the object
+        serializer_class = get_serializer_for_model(instance.__class__)
+        serializer_context = {
+            'request': None,
+        }
+        serializer = serializer_class(instance, context=serializer_context)
 
-    for webhook in webhooks:
+        # We must only import django_rq if the Webhooks feature is enabled.
-        webhook_queue.enqueue("extras.webhooks_worker.process_webhook",
+        # Only if we have gotten to ths point, is the feature enabled
-                              webhook,
+        from django_rq import get_queue
-                              serialized_data.data,
+        webhook_queue = get_queue('default')
-                              model_class,
-                              event,
-                              signal_received_timestamp)
 
-
+        # enqueue the webhooks:
-def post_save_receiver(sender, instance, created, **kwargs):
+        for webhook in webhooks:
-    """
+            webhook_queue.enqueue(
-    Receives post_save signals from registered models. If the webhook
+                "extras.webhooks_worker.process_webhook",
-    backend is enabled, queue any webhooks that apply to the event.
+                webhook,
-    """
+                serializer.data,
-    if settings.WEBHOOKS_ENABLED:
+                instance.__class__,
-        signal_received_timestamp = time.time()
+                action,
-        # look for any webhooks that match this event
+                str(datetime.datetime.now())
-        updated = not created
+            )
-        obj_type = ContentType.objects.get_for_model(sender)
-        webhooks = Webhook.objects.filter(
-            Q(enabled=True) &
-            (
-                Q(type_create=created) |
-                Q(type_update=updated)
-            ) &
-            Q(obj_type=obj_type)
-        )
-        event = 'created' if created else 'updated'
-        if webhooks:
-            enqueue_webhooks(webhooks, sender, instance, event, signal_received_timestamp)
-
-
-def post_delete_receiver(sender, instance, **kwargs):
-    """
-    Receives post_delete signals from registered models. If the webhook
-    backend is enabled, queue any webhooks that apply to the event.
-    """
-    if settings.WEBHOOKS_ENABLED:
-        signal_received_timestamp = time.time()
-        obj_type = ContentType.objects.get_for_model(sender)
-        # look for any webhooks that match this event
-        webhooks = Webhook.objects.filter(enabled=True, type_delete=True, obj_type=obj_type)
-        if webhooks:
-            enqueue_webhooks(webhooks, sender, instance, 'deleted', signal_received_timestamp)
-
-
-def bulk_operation_receiver(sender, **kwargs):
-    """
-    Receives bulk_operation_signal signals from registered models. If the webhook
-    backend is enabled, queue any webhooks that apply to the event.
-    """
-    if settings.WEBHOOKS_ENABLED:
-        signal_received_timestamp = time.time()
-        event = kwargs['event']
-        obj_type = ContentType.objects.get_for_model(sender)
-        # look for any webhooks that match this event
-        if event == 'created':
-            webhooks = Webhook.objects.filter(enabled=True, type_create=True, obj_type=obj_type)
-        elif event == 'updated':
-            webhooks = Webhook.objects.filter(enabled=True, type_update=True, obj_type=obj_type)
-        elif event == 'deleted':
-            webhooks = Webhook.objects.filter(enabled=True, type_delete=True, obj_type=obj_type)
-        else:
-            webhooks = None
-
-        if webhooks:
-            enqueue_webhooks(webhooks, sender, list(kwargs['instances']), event, signal_received_timestamp)
-
-
-# the bulk operation signal is used to overcome signals not being sent for bulk model changes
-bulk_operation_signal = Signal(providing_args=["instances", "event"])
-bulk_operation_signal.connect(bulk_operation_receiver)
-
-
-def register_signals(senders):
-    """
-    Take a list of senders (Models) and register them to the post_save
-    and post_delete signal receivers.
-    """
-    if settings.WEBHOOKS_ENABLED:
-        # only register signals if the backend is enabled
-        # this reduces load by not firing signals if the
-        # webhook backend feature is disabled
-
-        for sender in senders:
-            post_save.connect(post_save_receiver, sender=sender)
-            post_delete.connect(post_delete_receiver, sender=sender)

netbox/extras/webhooks_worker.py

@@ -4,7 +4,7 @@ import hmac
 import requests
 from django_rq import job
 
-from extras.constants import WEBHOOK_CT_JSON, WEBHOOK_CT_X_WWW_FORM_ENCODED
+from extras.constants import WEBHOOK_CT_JSON, WEBHOOK_CT_X_WWW_FORM_ENCODED, OBJECTCHANGE_ACTION_CHOICES
 
 
 @job('default')
@@ -13,7 +13,7 @@ def process_webhook(webhook, data, model_class, event, timestamp):
     Make a POST request to the defined Webhook
     """
     payload = {
-        'event': event,
+        'event': dict(OBJECTCHANGE_ACTION_CHOICES)[event],
         'timestamp': timestamp,
         'model': model_class.__name__,
         'data': data

netbox/ipam/apps.py

@@ -6,10 +6,3 @@ from django.apps import AppConfig
 class IPAMConfig(AppConfig):
     name = "ipam"
     verbose_name = "IPAM"
-
-    def ready(self):
-
-        # register webhook signals
-        from extras.webhooks import register_signals
-        from .models import Aggregate, Prefix, IPAddress, VLAN, VRF, VLANGroup, Service
-        register_signals([Aggregate, Prefix, IPAddress, VLAN, VRF, VLANGroup, Service])

netbox/ipam/models.py

@@ -59,7 +59,6 @@ class VRF(ChangeLoggedModel, CustomFieldModel):
 
     tags = TaggableManager()
 
-    serializer = 'ipam.api.serializers.VRFSerializer'
     csv_headers = ['name', 'rd', 'tenant', 'enforce_unique', 'description']
 
     class Meta:
@@ -108,7 +107,6 @@ class RIR(ChangeLoggedModel):
         help_text='IP space managed by this RIR is considered private'
     )
 
-    serializer = 'ipam.api.serializers.RIRSerializer'
     csv_headers = ['name', 'slug', 'is_private']
 
     class Meta:
@@ -162,7 +160,6 @@ class Aggregate(ChangeLoggedModel, CustomFieldModel):
 
     tags = TaggableManager()
 
-    serializer = 'ipam.api.serializers.AggregateSerializer'
     csv_headers = ['prefix', 'rir', 'date_added', 'description']
 
     class Meta:
@@ -243,7 +240,6 @@ class Role(ChangeLoggedModel):
         default=1000
     )
 
-    serializer = 'ipam.api.serializers.RoleSerializer'
     csv_headers = ['name', 'slug', 'weight']
 
     class Meta:
@@ -336,7 +332,6 @@ class Prefix(ChangeLoggedModel, CustomFieldModel):
     objects = PrefixQuerySet.as_manager()
     tags = TaggableManager()
 
-    serializer = 'ipam.api.serializers.PrefixSerializer'
     csv_headers = [
         'prefix', 'vrf', 'tenant', 'site', 'vlan_group', 'vlan_vid', 'status', 'role', 'is_pool', 'description',
     ]
@@ -577,7 +572,6 @@ class IPAddress(ChangeLoggedModel, CustomFieldModel):
     objects = IPAddressManager()
     tags = TaggableManager()
 
-    serializer = 'ipam.api.serializers.IPAddressSerializer'
     csv_headers = [
         'address', 'vrf', 'tenant', 'status', 'role', 'device', 'virtual_machine', 'interface_name', 'is_primary',
         'description',
@@ -677,7 +671,6 @@ class VLANGroup(ChangeLoggedModel):
         null=True
     )
 
-    serializer = 'ipam.api.serializers.VLANGroupSerializer'
     csv_headers = ['name', 'slug', 'site']
 
     class Meta:
@@ -775,7 +768,6 @@ class VLAN(ChangeLoggedModel, CustomFieldModel):
 
     tags = TaggableManager()
 
-    serializer = 'ipam.api.serializers.VLANSerializer'
     csv_headers = ['site', 'group_name', 'vid', 'name', 'tenant', 'status', 'role', 'description']
 
     class Meta:
@@ -879,7 +871,6 @@ class Service(ChangeLoggedModel, CustomFieldModel):
 
     tags = TaggableManager()
 
-    serializer = 'ipam.api.serializers.ServiceSerializer'
     csv_headers = ['device', 'virtual_machine', 'name', 'protocol', 'description']
 
     class Meta:

netbox/netbox/settings.py

@@ -175,7 +175,7 @@ MIDDLEWARE = (
     'utilities.middleware.ExceptionHandlingMiddleware',
     'utilities.middleware.LoginRequiredMiddleware',
     'utilities.middleware.APIVersionMiddleware',
-    'extras.middleware.ChangeLoggingMiddleware',
+    'extras.middleware.ObjectChangeMiddleware',
 )
 
 ROOT_URLCONF = 'netbox.urls'

netbox/secrets/models.py

@@ -285,7 +285,6 @@ class SecretRole(ChangeLoggedModel):
         blank=True
     )
 
-    serializer = 'ipam.api.secrets.SecretSerializer'
     csv_headers = ['name', 'slug']
 
     class Meta:
@@ -354,7 +353,6 @@ class Secret(ChangeLoggedModel, CustomFieldModel):
     tags = TaggableManager()
 
     plaintext = None
-    serializer = 'ipam.api.secrets.SecretSerializer'
     csv_headers = ['device', 'role', 'name', 'plaintext']
 
     class Meta:

netbox/tenancy/apps.py

@@ -5,10 +5,3 @@ from django.apps import AppConfig
 
 class TenancyConfig(AppConfig):
     name = 'tenancy'
-
-    def ready(self):
-
-        # register webhook signals
-        from extras.webhooks import register_signals
-        from .models import Tenant, TenantGroup
-        register_signals([Tenant, TenantGroup])

netbox/tenancy/models.py

@@ -23,7 +23,6 @@ class TenantGroup(ChangeLoggedModel):
         unique=True
     )
 
-    serializer = 'tenancy.api.serializers.TenantGroupSerializer'
     csv_headers = ['name', 'slug']
 
     class Meta:
@@ -78,7 +77,6 @@ class Tenant(ChangeLoggedModel, CustomFieldModel):
 
     tags = TaggableManager()
 
-    serializer = 'tenancy.api.serializers.TenantSerializer'
     csv_headers = ['name', 'slug', 'group', 'description', 'comments']
 
     class Meta:

netbox/utilities/views.py

@@ -25,7 +25,6 @@ from django.views.generic import View
 from django_tables2 import RequestConfig
 
 from extras.models import CustomField, CustomFieldValue, ExportTemplate
-from extras.webhooks import bulk_operation_signal
 from utilities.utils import queryset_to_csv
 from utilities.forms import BootstrapMixin, CSVDataField
 from .constants import M2M_FIELD_TYPES
@@ -757,9 +756,6 @@ class ComponentCreateView(View):
                                 field_links.append(field_link)
                         getattr(self.model, field).through.objects.bulk_create(field_links)
 
-                # send the bulk operations signal for webhooks
-                bulk_operation_signal.send(sender=self.model, instances=new_components, event="created")
-
                 messages.success(request, "Added {} {} to {}.".format(
                     len(new_components), self.model._meta.verbose_name_plural, parent
                 ))
@@ -829,9 +825,6 @@ class BulkComponentCreateView(GetReturnURLMixin, View):
                 if not form.errors:
                     self.model.objects.bulk_create(new_components)
 
-                    # send the bulk operations signal for webhooks
-                    bulk_operation_signal.send(sender=self.model, instances=new_components, event="created")
-
                     messages.success(request, "Added {} {} to {} {}.".format(
                         len(new_components),
                         self.model._meta.verbose_name_plural,

netbox/virtualization/apps.py

@@ -5,10 +5,3 @@ from django.apps import AppConfig
 
 class VirtualizationConfig(AppConfig):
     name = 'virtualization'
-
-    def ready(self):
-
-        # register webhook signals
-        from extras.webhooks import register_signals
-        from .models import Cluster, ClusterGroup, VirtualMachine
-        register_signals([Cluster, VirtualMachine, ClusterGroup])

netbox/virtualization/models.py

@@ -31,7 +31,6 @@ class ClusterType(ChangeLoggedModel):
         unique=True
     )
 
-    serializer = 'virtualization.api.serializers.ClusterTypeSerializer'
     csv_headers = ['name', 'slug']
 
     class Meta:
@@ -67,7 +66,6 @@ class ClusterGroup(ChangeLoggedModel):
         unique=True
     )
 
-    serializer = 'virtualization.api.serializers.ClusterGroupSerializer'
     csv_headers = ['name', 'slug']
 
     class Meta:
@@ -129,7 +127,6 @@ class Cluster(ChangeLoggedModel, CustomFieldModel):
 
     tags = TaggableManager()
 
-    serializer = 'virtualization.api.serializers.ClusterSerializer'
     csv_headers = ['name', 'type', 'group', 'site', 'comments']
 
     class Meta:
@@ -250,7 +247,6 @@ class VirtualMachine(ChangeLoggedModel, ConfigContextModel, CustomFieldModel):
 
     tags = TaggableManager()
 
-    serializer = 'virtualization.api.serializers.VirtualMachineSerializer'
     csv_headers = [
         'name', 'status', 'role', 'cluster', 'tenant', 'platform', 'vcpus', 'memory', 'disk', 'comments',
     ]