mirror of
https://github.com/netbox-community/netbox.git
synced 2024-05-10 07:54:54 +00:00
Initial work on #6284
This commit is contained in:
netbox/extras
@ -4,6 +4,7 @@ from django.db.models.signals import m2m_changed, pre_delete, post_save
|
||||
|
||||
from extras.signals import _handle_changed_object, _handle_deleted_object
|
||||
from utilities.utils import curry
|
||||
from .webhooks import flush_webhooks
|
||||
|
||||
|
||||
@contextmanager
|
||||
@ -14,9 +15,11 @@ def change_logging(request):
|
||||
|
||||
:param request: WSGIRequest object with a unique `id` set
|
||||
"""
|
||||
webhook_queue = []
|
||||
|
||||
# Curry signals receivers to pass the current request
|
||||
handle_changed_object = curry(_handle_changed_object, request)
|
||||
handle_deleted_object = curry(_handle_deleted_object, request)
|
||||
handle_changed_object = curry(_handle_changed_object, request, webhook_queue)
|
||||
handle_deleted_object = curry(_handle_deleted_object, request, webhook_queue)
|
||||
|
||||
# Connect our receivers to the post_save and post_delete signals.
|
||||
post_save.connect(handle_changed_object, dispatch_uid='handle_changed_object')
|
||||
@ -30,3 +33,7 @@ def change_logging(request):
|
||||
post_save.disconnect(handle_changed_object, dispatch_uid='handle_changed_object')
|
||||
m2m_changed.disconnect(handle_changed_object, dispatch_uid='handle_changed_object')
|
||||
pre_delete.disconnect(handle_deleted_object, dispatch_uid='handle_deleted_object')
|
||||
|
||||
# Flush queued webhooks to RQ
|
||||
flush_webhooks(webhook_queue)
|
||||
del webhook_queue
|
||||
|
@ -12,17 +12,20 @@ from prometheus_client import Counter
|
||||
|
||||
from .choices import ObjectChangeActionChoices
|
||||
from .models import CustomField, ObjectChange
|
||||
from .webhooks import enqueue_webhooks
|
||||
from .webhooks import enqueue_object, serialize_for_webhook
|
||||
|
||||
|
||||
#
|
||||
# Change logging/webhooks
|
||||
#
|
||||
|
||||
def _handle_changed_object(request, sender, instance, **kwargs):
|
||||
def _handle_changed_object(request, webhook_queue, sender, instance, **kwargs):
|
||||
"""
|
||||
Fires when an object is created or updated.
|
||||
"""
|
||||
if not hasattr(instance, 'to_objectchange'):
|
||||
return
|
||||
|
||||
m2m_changed = False
|
||||
|
||||
# Determine the type of change being made
|
||||
@ -53,8 +56,13 @@ def _handle_changed_object(request, sender, instance, **kwargs):
|
||||
objectchange.request_id = request.id
|
||||
objectchange.save()
|
||||
|
||||
# Enqueue webhooks
|
||||
enqueue_webhooks(instance, request.user, request.id, action)
|
||||
# If this is an M2M change, update the previously queued webhook (from post_save)
|
||||
if m2m_changed and webhook_queue:
|
||||
# TODO: Need more validation here
|
||||
# TODO: Need to account for snapshot changes
|
||||
webhook_queue[-1]['data'] = serialize_for_webhook(instance)
|
||||
else:
|
||||
enqueue_object(webhook_queue, instance, request.user, request.id, action)
|
||||
|
||||
# Increment metric counters
|
||||
if action == ObjectChangeActionChoices.ACTION_CREATE:
|
||||
@ -68,10 +76,13 @@ def _handle_changed_object(request, sender, instance, **kwargs):
|
||||
ObjectChange.objects.filter(time__lt=cutoff)._raw_delete(using=DEFAULT_DB_ALIAS)
|
||||
|
||||
|
||||
def _handle_deleted_object(request, sender, instance, **kwargs):
|
||||
def _handle_deleted_object(request, webhook_queue, sender, instance, **kwargs):
|
||||
"""
|
||||
Fires when an object is deleted.
|
||||
"""
|
||||
if not hasattr(instance, 'to_objectchange'):
|
||||
return
|
||||
|
||||
# Record an ObjectChange if applicable
|
||||
if hasattr(instance, 'to_objectchange'):
|
||||
objectchange = instance.to_objectchange(ObjectChangeActionChoices.ACTION_DELETE)
|
||||
@ -80,7 +91,7 @@ def _handle_deleted_object(request, sender, instance, **kwargs):
|
||||
objectchange.save()
|
||||
|
||||
# Enqueue webhooks
|
||||
enqueue_webhooks(instance, request.user, request.id, ObjectChangeActionChoices.ACTION_DELETE)
|
||||
enqueue_object(webhook_queue, instance, request.user, request.id, ObjectChangeActionChoices.ACTION_DELETE)
|
||||
|
||||
# Increment metric counters
|
||||
model_deletes.labels(instance._meta.model_name).inc()
|
||||
|
@ -12,7 +12,7 @@ from rest_framework import status
|
||||
from dcim.models import Site
|
||||
from extras.choices import ObjectChangeActionChoices
|
||||
from extras.models import Webhook
|
||||
from extras.webhooks import enqueue_webhooks, generate_signature
|
||||
from extras.webhooks import enqueue_object, generate_signature
|
||||
from extras.webhooks_worker import process_webhook
|
||||
from utilities.testing import APITestCase
|
||||
|
||||
@ -96,46 +96,48 @@ class WebhookTest(APITestCase):
|
||||
self.assertEqual(job.kwargs['model_name'], 'site')
|
||||
self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE)
|
||||
|
||||
def test_webhooks_worker(self):
|
||||
|
||||
request_id = uuid.uuid4()
|
||||
|
||||
def dummy_send(_, request, **kwargs):
|
||||
"""
|
||||
A dummy implementation of Session.send() to be used for testing.
|
||||
Always returns a 200 HTTP response.
|
||||
"""
|
||||
webhook = Webhook.objects.get(type_create=True)
|
||||
signature = generate_signature(request.body, webhook.secret)
|
||||
|
||||
# Validate the outgoing request headers
|
||||
self.assertEqual(request.headers['Content-Type'], webhook.http_content_type)
|
||||
self.assertEqual(request.headers['X-Hook-Signature'], signature)
|
||||
self.assertEqual(request.headers['X-Foo'], 'Bar')
|
||||
|
||||
# Validate the outgoing request body
|
||||
body = json.loads(request.body)
|
||||
self.assertEqual(body['event'], 'created')
|
||||
self.assertEqual(body['timestamp'], job.kwargs['timestamp'])
|
||||
self.assertEqual(body['model'], 'site')
|
||||
self.assertEqual(body['username'], 'testuser')
|
||||
self.assertEqual(body['request_id'], str(request_id))
|
||||
self.assertEqual(body['data']['name'], 'Site 1')
|
||||
|
||||
return HttpResponse()
|
||||
|
||||
# Enqueue a webhook for processing
|
||||
site = Site.objects.create(name='Site 1', slug='site-1')
|
||||
enqueue_webhooks(
|
||||
instance=site,
|
||||
user=self.user,
|
||||
request_id=request_id,
|
||||
action=ObjectChangeActionChoices.ACTION_CREATE
|
||||
)
|
||||
|
||||
# Retrieve the job from queue
|
||||
job = self.queue.jobs[0]
|
||||
|
||||
# Patch the Session object with our dummy_send() method, then process the webhook for sending
|
||||
with patch.object(Session, 'send', dummy_send) as mock_send:
|
||||
process_webhook(**job.kwargs)
|
||||
# TODO: Replace webhook worker test
|
||||
# def test_webhooks_worker(self):
|
||||
#
|
||||
# request_id = uuid.uuid4()
|
||||
#
|
||||
# def dummy_send(_, request, **kwargs):
|
||||
# """
|
||||
# A dummy implementation of Session.send() to be used for testing.
|
||||
# Always returns a 200 HTTP response.
|
||||
# """
|
||||
# webhook = Webhook.objects.get(type_create=True)
|
||||
# signature = generate_signature(request.body, webhook.secret)
|
||||
#
|
||||
# # Validate the outgoing request headers
|
||||
# self.assertEqual(request.headers['Content-Type'], webhook.http_content_type)
|
||||
# self.assertEqual(request.headers['X-Hook-Signature'], signature)
|
||||
# self.assertEqual(request.headers['X-Foo'], 'Bar')
|
||||
#
|
||||
# # Validate the outgoing request body
|
||||
# body = json.loads(request.body)
|
||||
# self.assertEqual(body['event'], 'created')
|
||||
# self.assertEqual(body['timestamp'], job.kwargs['timestamp'])
|
||||
# self.assertEqual(body['model'], 'site')
|
||||
# self.assertEqual(body['username'], 'testuser')
|
||||
# self.assertEqual(body['request_id'], str(request_id))
|
||||
# self.assertEqual(body['data']['name'], 'Site 1')
|
||||
#
|
||||
# return HttpResponse()
|
||||
#
|
||||
# # Enqueue a webhook for processing
|
||||
# site = Site.objects.create(name='Site 1', slug='site-1')
|
||||
# enqueue_webhooks(
|
||||
# queue=[],
|
||||
# instance=site,
|
||||
# user=self.user,
|
||||
# request_id=request_id,
|
||||
# action=ObjectChangeActionChoices.ACTION_CREATE
|
||||
# )
|
||||
#
|
||||
# # Retrieve the job from queue
|
||||
# job = self.queue.jobs[0]
|
||||
#
|
||||
# # Patch the Session object with our dummy_send() method, then process the webhook for sending
|
||||
# with patch.object(Session, 'send', dummy_send) as mock_send:
|
||||
# process_webhook(**job.kwargs)
|
||||
|
@ -12,6 +12,19 @@ from .models import Webhook
|
||||
from .registry import registry
|
||||
|
||||
|
||||
def serialize_for_webhook(instance):
|
||||
"""
|
||||
Return a serialized representation of the given instance suitable for use in a webhook.
|
||||
"""
|
||||
serializer_class = get_serializer_for_model(instance.__class__)
|
||||
serializer_context = {
|
||||
'request': None,
|
||||
}
|
||||
serializer = serializer_class(instance, context=serializer_context)
|
||||
|
||||
return serializer.data
|
||||
|
||||
|
||||
def generate_signature(request_body, secret):
|
||||
"""
|
||||
Return a cryptographic signature that can be used to verify the authenticity of webhook data.
|
||||
@ -24,10 +37,10 @@ def generate_signature(request_body, secret):
|
||||
return hmac_prep.hexdigest()
|
||||
|
||||
|
||||
def enqueue_webhooks(instance, user, request_id, action):
|
||||
def enqueue_object(queue, instance, user, request_id, action):
|
||||
"""
|
||||
Find Webhook(s) assigned to this instance + action and enqueue them
|
||||
to be processed
|
||||
Enqueue a serialized representation of a created/updated/deleted object for the processing of
|
||||
webhooks once the request has completed.
|
||||
"""
|
||||
# Determine whether this type of object supports webhooks
|
||||
app_label = instance._meta.app_label
|
||||
@ -35,41 +48,50 @@ def enqueue_webhooks(instance, user, request_id, action):
|
||||
if model_name not in registry['model_features']['webhooks'].get(app_label, []):
|
||||
return
|
||||
|
||||
# Retrieve any applicable Webhooks
|
||||
content_type = ContentType.objects.get_for_model(instance)
|
||||
action_flag = {
|
||||
ObjectChangeActionChoices.ACTION_CREATE: 'type_create',
|
||||
ObjectChangeActionChoices.ACTION_UPDATE: 'type_update',
|
||||
ObjectChangeActionChoices.ACTION_DELETE: 'type_delete',
|
||||
}[action]
|
||||
webhooks = Webhook.objects.filter(content_types=content_type, enabled=True, **{action_flag: True})
|
||||
# Gather pre- and post-change snapshots
|
||||
snapshots = {
|
||||
'prechange': getattr(instance, '_prechange_snapshot', None),
|
||||
'postchange': serialize_object(instance) if action != ObjectChangeActionChoices.ACTION_DELETE else None,
|
||||
}
|
||||
|
||||
if webhooks.exists():
|
||||
queue.append({
|
||||
'content_type': ContentType.objects.get_for_model(instance),
|
||||
'object_id': instance.pk,
|
||||
'event': action,
|
||||
'data': serialize_for_webhook(instance),
|
||||
'snapshots': snapshots,
|
||||
'username': user.username,
|
||||
'request_id': request_id
|
||||
})
|
||||
|
||||
# Get the Model's API serializer class and serialize the object
|
||||
serializer_class = get_serializer_for_model(instance.__class__)
|
||||
serializer_context = {
|
||||
'request': None,
|
||||
}
|
||||
serializer = serializer_class(instance, context=serializer_context)
|
||||
|
||||
# Gather pre- and post-change snapshots
|
||||
snapshots = {
|
||||
'prechange': getattr(instance, '_prechange_snapshot', None),
|
||||
'postchange': serialize_object(instance) if action != ObjectChangeActionChoices.ACTION_DELETE else None,
|
||||
}
|
||||
def flush_webhooks(queue):
|
||||
"""
|
||||
Flush a list of object representation to RQ for webhook processing.
|
||||
"""
|
||||
rq_queue = get_queue('default')
|
||||
|
||||
for data in queue:
|
||||
|
||||
# Collect Webhooks that apply for this object and action
|
||||
content_type = data['content_type']
|
||||
action_flag = {
|
||||
ObjectChangeActionChoices.ACTION_CREATE: 'type_create',
|
||||
ObjectChangeActionChoices.ACTION_UPDATE: 'type_update',
|
||||
ObjectChangeActionChoices.ACTION_DELETE: 'type_delete',
|
||||
}[data['event']]
|
||||
# TODO: Cache these so we're not calling multiple times for bulk operations
|
||||
webhooks = Webhook.objects.filter(content_types=content_type, enabled=True, **{action_flag: True})
|
||||
|
||||
# Enqueue the webhooks
|
||||
webhook_queue = get_queue('default')
|
||||
for webhook in webhooks:
|
||||
webhook_queue.enqueue(
|
||||
rq_queue.enqueue(
|
||||
"extras.webhooks_worker.process_webhook",
|
||||
webhook=webhook,
|
||||
model_name=instance._meta.model_name,
|
||||
event=action,
|
||||
data=serializer.data,
|
||||
snapshots=snapshots,
|
||||
model_name=content_type.model,
|
||||
event=data['event'],
|
||||
data=data['data'],
|
||||
snapshots=data['snapshots'],
|
||||
timestamp=str(timezone.now()),
|
||||
username=user.username,
|
||||
request_id=request_id
|
||||
username=data['username'],
|
||||
request_id=data['request_id']
|
||||
)
|
||||
|
Reference in New Issue
Block a user