mirror/netbox-community-netbox
1
0
Fork 0
mirror of https://github.com/netbox-community/netbox.git synced 2024-05-10 07:54:54 +00:00
Code Activity

Webhook signal refactor - fixes #2282 (#2260)

Browse Source 
Refactor of webhook signaling system to use the same middleware mechanics of Changelogging
This commit is contained in:
John Anderson
2018-07-30 14:23:49 -04:00
committed by Jeremy Stretch
parent 9876a2efcd
commit 722d0d5554
17 changed files with 65 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
netbox/extras/middleware.py
View File

@@ -9,7 +9,11 @@ from django.conf import settings
from django.db.models.signals import post_delete, post_save
from django.utils import timezone
from .constants import OBJECTCHANGE_ACTION_CREATE, OBJECTCHANGE_ACTION_DELETE, OBJECTCHANGE_ACTION_UPDATE
from extras.webhooks import enqueue_webhooks
from .constants import (
OBJECTCHANGE_ACTION_CREATE, OBJECTCHANGE_ACTION_DELETE, OBJECTCHANGE_ACTION_UPDATE,
WEBHOOK_MODELS
)
from .models import ObjectChange
@@ -18,12 +22,10 @@ _thread_locals = threading.local()
def mark_object_changed(instance, **kwargs):
"""
Mark an object as having been created, saved, or updated. At the end of the request, this change will be recorded.
We have to wait until the *end* of the request to the serialize the object, because related fields like tags and
custom fields have not yet been updated when the post_save signal is emitted.
Mark an object as having been created, saved, or updated. At the end of the request, this change will be recorded
and/or associated webhooks fired. We have to wait until the *end* of the request to the serialize the object,
because related fields like tags and custom fields have not yet been updated when the post_save signal is emitted.
"""
if not hasattr(instance, 'log_change'):
return
# Determine what action is being performed. The post_save signal sends a `created` boolean, whereas post_delete
# does not.
@@ -35,7 +37,12 @@ def mark_object_changed(instance, **kwargs):
_thread_locals.changed_objects.append((instance, action))
class ChangeLoggingMiddleware(object):
class ObjectChangeMiddleware(object):
"""
This middleware intercepts all requests to connects object signals to the Django runtime. The signals collect all
changed objects into a local thread by way of the `mark_object_changed()` receiver. At the end of the request,
the middleware iterates over the objects to process change events like Change Logging and Webhooks.
"""
def __init__(self, get_response):
self.get_response = get_response
@@ -56,11 +63,16 @@ class ChangeLoggingMiddleware(object):
# Process the request
response = self.get_response(request)
# Record object changes
# Perform change logging and fire Webhook signals
for obj, action in _thread_locals.changed_objects:
if obj.pk:
# Log object changes
if obj.pk and hasattr(obj, 'log_change'):
obj.log_change(request.user, request.id, action)
# Enqueue Webhooks if they are enabled
if settings.WEBHOOKS_ENABLED and obj.__class__.__name__.lower() in WEBHOOK_MODELS:
enqueue_webhooks(obj, action)
# Housekeeping: 1% chance of clearing out expired ObjectChanges
if _thread_locals.changed_objects and settings.CHANGELOG_RETENTION and random.randint(1, 100) == 1:
cutoff = timezone.now() - timedelta(days=settings.CHANGELOG_RETENTION)

1
netbox/extras/models.py
View File

@@ -813,7 +813,6 @@ class ObjectChange(models.Model):
editable=False
)
serializer = 'extras.api.serializers.ObjectChangeSerializer'
csv_headers = [
'time', 'user', 'user_name', 'request_id', 'action', 'changed_object_type', 'changed_object_id',
'related_object_type', 'related_object_id', 'object_repr', 'object_data',

147
netbox/extras/webhooks.py
View File

@@ -1,119 +1,54 @@
import time
import datetime
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
from django.db.models.signals import post_save, post_delete
from django.dispatch import Signal
from extras.models import Webhook
from utilities.utils import dynamic_import
from extras.constants import OBJECTCHANGE_ACTION_CREATE, OBJECTCHANGE_ACTION_DELETE, OBJECTCHANGE_ACTION_UPDATE
from utilities.api import get_serializer_for_model
def enqueue_webhooks(webhooks, model_class, data, event, signal_received_timestamp):
def enqueue_webhooks(instance, action):
"""
Serialize data and enqueue webhooks
Find Webhook(s) assigned to this instance + action and enqueue them
to be processed
"""
serializer_context = {
'request': None,
}
type_create = action == OBJECTCHANGE_ACTION_CREATE
type_update = action == OBJECTCHANGE_ACTION_UPDATE
type_delete = action == OBJECTCHANGE_ACTION_DELETE
if isinstance(data, list):
serializer_property = data[0].serializer
serializer_cls = dynamic_import(serializer_property)
serialized_data = serializer_cls(data, context=serializer_context, many=True)
else:
serializer_property = data.serializer
serializer_cls = dynamic_import(serializer_property)
serialized_data = serializer_cls(data, context=serializer_context)
# Find assigned webhooks
obj_type = ContentType.objects.get_for_model(instance.__class__)
webhooks = Webhook.objects.filter(
Q(enabled=True) &
(
Q(type_create=type_create) |
Q(type_update=type_update) |
Q(type_delete=type_delete)
) &
Q(obj_type=obj_type)
)
from django_rq import get_queue
webhook_queue = get_queue('default')
if webhooks:
# Get the Model's API serializer class and serialize the object
serializer_class = get_serializer_for_model(instance.__class__)
serializer_context = {
'request': None,
}
serializer = serializer_class(instance, context=serializer_context)
for webhook in webhooks:
webhook_queue.enqueue("extras.webhooks_worker.process_webhook",
webhook,
serialized_data.data,
model_class,
event,
signal_received_timestamp)
# We must only import django_rq if the Webhooks feature is enabled.
# Only if we have gotten to ths point, is the feature enabled
from django_rq import get_queue
webhook_queue = get_queue('default')
def post_save_receiver(sender, instance, created, **kwargs):
"""
Receives post_save signals from registered models. If the webhook
backend is enabled, queue any webhooks that apply to the event.
"""
if settings.WEBHOOKS_ENABLED:
signal_received_timestamp = time.time()
# look for any webhooks that match this event
updated = not created
obj_type = ContentType.objects.get_for_model(sender)
webhooks = Webhook.objects.filter(
Q(enabled=True) &
(
Q(type_create=created) |
Q(type_update=updated)
) &
Q(obj_type=obj_type)
)
event = 'created' if created else 'updated'
if webhooks:
enqueue_webhooks(webhooks, sender, instance, event, signal_received_timestamp)
def post_delete_receiver(sender, instance, **kwargs):
"""
Receives post_delete signals from registered models. If the webhook
backend is enabled, queue any webhooks that apply to the event.
"""
if settings.WEBHOOKS_ENABLED:
signal_received_timestamp = time.time()
obj_type = ContentType.objects.get_for_model(sender)
# look for any webhooks that match this event
webhooks = Webhook.objects.filter(enabled=True, type_delete=True, obj_type=obj_type)
if webhooks:
enqueue_webhooks(webhooks, sender, instance, 'deleted', signal_received_timestamp)
def bulk_operation_receiver(sender, **kwargs):
"""
Receives bulk_operation_signal signals from registered models. If the webhook
backend is enabled, queue any webhooks that apply to the event.
"""
if settings.WEBHOOKS_ENABLED:
signal_received_timestamp = time.time()
event = kwargs['event']
obj_type = ContentType.objects.get_for_model(sender)
# look for any webhooks that match this event
if event == 'created':
webhooks = Webhook.objects.filter(enabled=True, type_create=True, obj_type=obj_type)
elif event == 'updated':
webhooks = Webhook.objects.filter(enabled=True, type_update=True, obj_type=obj_type)
elif event == 'deleted':
webhooks = Webhook.objects.filter(enabled=True, type_delete=True, obj_type=obj_type)
else:
webhooks = None
if webhooks:
enqueue_webhooks(webhooks, sender, list(kwargs['instances']), event, signal_received_timestamp)
# the bulk operation signal is used to overcome signals not being sent for bulk model changes
bulk_operation_signal = Signal(providing_args=["instances", "event"])
bulk_operation_signal.connect(bulk_operation_receiver)
def register_signals(senders):
"""
Take a list of senders (Models) and register them to the post_save
and post_delete signal receivers.
"""
if settings.WEBHOOKS_ENABLED:
# only register signals if the backend is enabled
# this reduces load by not firing signals if the
# webhook backend feature is disabled
for sender in senders:
post_save.connect(post_save_receiver, sender=sender)
post_delete.connect(post_delete_receiver, sender=sender)
# enqueue the webhooks:
for webhook in webhooks:
webhook_queue.enqueue(
"extras.webhooks_worker.process_webhook",
webhook,
serializer.data,
instance.__class__,
action,
str(datetime.datetime.now())
)

4
netbox/extras/webhooks_worker.py
View File

@@ -4,7 +4,7 @@ import hmac
import requests
from django_rq import job
from extras.constants import WEBHOOK_CT_JSON, WEBHOOK_CT_X_WWW_FORM_ENCODED
from extras.constants import WEBHOOK_CT_JSON, WEBHOOK_CT_X_WWW_FORM_ENCODED, OBJECTCHANGE_ACTION_CHOICES
@job('default')
@@ -13,7 +13,7 @@ def process_webhook(webhook, data, model_class, event, timestamp):
Make a POST request to the defined Webhook
"""
payload = {
'event': event,
'event': dict(OBJECTCHANGE_ACTION_CHOICES)[event],
'timestamp': timestamp,
'model': model_class.__name__,
'data': data