mirror/netbox-community-netbox
1
0
Fork 0
mirror of https://github.com/netbox-community/netbox.git synced 2024-05-10 07:54:54 +00:00
Code Activity

Webhook signal refactor - fixes #2282 (#2260)

Browse Source 
Refactor of webhook signaling system to use the same middleware mechanics of Changelogging
This commit is contained in:
John Anderson
2018-07-30 14:23:49 -04:00
committed by Jeremy Stretch
parent 9876a2efcd
commit 722d0d5554
17 changed files with 65 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
netbox/circuits/apps.py
View File

@ -9,8 +9,3 @@ class CircuitsConfig(AppConfig):
def ready(self):
import circuits.signals
# register webhook signals
from extras.webhooks import register_signals
from .models import Circuit, Provider
register_signals([Circuit, Provider])

3
netbox/circuits/models.py
View File

@ -60,7 +60,6 @@ class Provider(ChangeLoggedModel, CustomFieldModel):
tags = TaggableManager()
serializer = 'circuits.api.serializers.ProviderSerializer'
csv_headers = ['name', 'slug', 'asn', 'account', 'portal_url', 'noc_contact', 'admin_contact', 'comments']
class Meta:
@ -99,7 +98,6 @@ class CircuitType(ChangeLoggedModel):
unique=True
)
serializer = 'circuits.api.serializers.CircuitTypeSerializer'
csv_headers = ['name', 'slug']
class Meta:
@ -174,7 +172,6 @@ class Circuit(ChangeLoggedModel, CustomFieldModel):
tags = TaggableManager()
serializer = 'circuits.api.serializers.CircuitSerializer'
csv_headers = [
'cid', 'provider', 'type', 'status', 'tenant', 'install_date', 'commit_rate', 'description', 'comments',
]

5
netbox/dcim/apps.py
View File

@ -10,8 +10,3 @@ class DCIMConfig(AppConfig):
def ready(self):
import dcim.signals
# register webhook signals
from extras.webhooks import register_signals
from .models import Site, Rack, RackGroup, Device, Interface
register_signals([Site, Rack, Device, Interface, RackGroup])

15
netbox/dcim/models.py
View File

@ -79,7 +79,6 @@ class Region(MPTTModel, ChangeLoggedModel):
unique=True
)
serializer = 'dcim.api.serializers.RegionSerializer'
csv_headers = ['name', 'slug', 'parent']
class MPTTMeta:
@ -201,7 +200,6 @@ class Site(ChangeLoggedModel, CustomFieldModel):
objects = SiteManager()
tags = TaggableManager()
serializer = 'dcim.api.serializers.SiteSerializer'
csv_headers = [
'name', 'slug', 'status', 'region', 'tenant', 'facility', 'asn', 'time_zone', 'description', 'physical_address',
'shipping_address', 'latitude', 'longitude', 'contact_name', 'contact_phone', 'contact_email', 'comments',
@ -287,7 +285,6 @@ class RackGroup(ChangeLoggedModel):
related_name='rack_groups'
)
serializer = 'dcim.api.serializers.RackGroupSerializer'
csv_headers = ['site', 'name', 'slug']
class Meta:
@ -325,7 +322,6 @@ class RackRole(ChangeLoggedModel):
)
color = ColorField()
serializer = 'dcim.api.serializers.RackRoleSerializer'
csv_headers = ['name', 'slug', 'color']
class Meta:
@ -432,7 +428,6 @@ class Rack(ChangeLoggedModel, CustomFieldModel):
objects = RackManager()
tags = TaggableManager()
serializer = 'dcim.api.serializers.RackSerializer'
csv_headers = [
'site', 'group_name', 'name', 'facility_id', 'tenant', 'role', 'type', 'serial', 'width', 'u_height',
'desc_units', 'comments',
@ -636,8 +631,6 @@ class RackReservation(ChangeLoggedModel):
max_length=100
)
serializer = 'dcim.api.serializers.RackReservationSerializer'
class Meta:
ordering = ['created']
@ -697,7 +690,6 @@ class Manufacturer(ChangeLoggedModel):
unique=True
)
serializer = 'dcim.api.serializers.ManufacturerSerializer'
csv_headers = ['name', 'slug']
class Meta:
@ -792,7 +784,6 @@ class DeviceType(ChangeLoggedModel, CustomFieldModel):
tags = TaggableManager()
serializer = 'dcim.api.serializers.DeviceTypeSerializer'
csv_headers = [
'manufacturer', 'model', 'slug', 'part_number', 'u_height', 'is_full_depth', 'is_console_server',
'is_pdu', 'is_network_device', 'subdevice_role', 'interface_ordering', 'comments',
@ -1076,7 +1067,6 @@ class DeviceRole(ChangeLoggedModel):
help_text='Virtual machines may be assigned to this role'
)
serializer = 'dcim.api.serializers.DeviceRoleSerializer'
csv_headers = ['name', 'slug', 'color', 'vm_role']
class Meta:
@ -1135,7 +1125,6 @@ class Platform(ChangeLoggedModel):
verbose_name='Legacy RPC client'
)
serializer = 'dcim.api.serializers.PlatformSerializer'
csv_headers = ['name', 'slug', 'manufacturer', 'napalm_driver', 'napalm_args']
class Meta:
@ -1302,7 +1291,6 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel):
objects = DeviceManager()
tags = TaggableManager()
serializer = 'dcim.api.serializers.DeviceSerializer'
csv_headers = [
'name', 'device_role', 'tenant', 'manufacturer', 'model_name', 'platform', 'serial', 'asset_tag', 'status',
'site', 'rack_group', 'rack_name', 'position', 'face', 'comments',
@ -1858,8 +1846,6 @@ class Interface(ComponentModel):
objects = InterfaceQuerySet.as_manager()
tags = TaggableManager()
serializer = 'dcim.api.serializers.InterfaceSerializer'
class Meta:
ordering = ['device', 'name']
unique_together = ['device', 'name']
@ -2263,7 +2249,6 @@ class VirtualChassis(ChangeLoggedModel):
tags = TaggableManager()
serializer = 'dcim.api.serializers.VirtualChassisSerializer'
csv_headers = ['master', 'domain']
class Meta:

30
netbox/extras/middleware.py
View File

@ -9,7 +9,11 @@ from django.conf import settings
from django.db.models.signals import post_delete, post_save
from django.utils import timezone
from .constants import OBJECTCHANGE_ACTION_CREATE, OBJECTCHANGE_ACTION_DELETE, OBJECTCHANGE_ACTION_UPDATE
from extras.webhooks import enqueue_webhooks
from .constants import (
OBJECTCHANGE_ACTION_CREATE, OBJECTCHANGE_ACTION_DELETE, OBJECTCHANGE_ACTION_UPDATE,
WEBHOOK_MODELS
)
from .models import ObjectChange
@ -18,12 +22,10 @@ _thread_locals = threading.local()
def mark_object_changed(instance, **kwargs):
"""
Mark an object as having been created, saved, or updated. At the end of the request, this change will be recorded.
We have to wait until the *end* of the request to the serialize the object, because related fields like tags and
custom fields have not yet been updated when the post_save signal is emitted.
Mark an object as having been created, saved, or updated. At the end of the request, this change will be recorded
and/or associated webhooks fired. We have to wait until the *end* of the request to the serialize the object,
because related fields like tags and custom fields have not yet been updated when the post_save signal is emitted.
"""
if not hasattr(instance, 'log_change'):
return
# Determine what action is being performed. The post_save signal sends a `created` boolean, whereas post_delete
# does not.
@ -35,7 +37,12 @@ def mark_object_changed(instance, **kwargs):
_thread_locals.changed_objects.append((instance, action))
class ChangeLoggingMiddleware(object):
class ObjectChangeMiddleware(object):
"""
This middleware intercepts all requests to connects object signals to the Django runtime. The signals collect all
changed objects into a local thread by way of the `mark_object_changed()` receiver. At the end of the request,
the middleware iterates over the objects to process change events like Change Logging and Webhooks.
"""
def __init__(self, get_response):
self.get_response = get_response
@ -56,11 +63,16 @@ class ChangeLoggingMiddleware(object):
# Process the request
response = self.get_response(request)
# Record object changes
# Perform change logging and fire Webhook signals
for obj, action in _thread_locals.changed_objects:
if obj.pk:
# Log object changes
if obj.pk and hasattr(obj, 'log_change'):
obj.log_change(request.user, request.id, action)
# Enqueue Webhooks if they are enabled
if settings.WEBHOOKS_ENABLED and obj.__class__.__name__.lower() in WEBHOOK_MODELS:
enqueue_webhooks(obj, action)
# Housekeeping: 1% chance of clearing out expired ObjectChanges
if _thread_locals.changed_objects and settings.CHANGELOG_RETENTION and random.randint(1, 100) == 1:
cutoff = timezone.now() - timedelta(days=settings.CHANGELOG_RETENTION)

1
netbox/extras/models.py
View File

@ -813,7 +813,6 @@ class ObjectChange(models.Model):
editable=False
)
serializer = 'extras.api.serializers.ObjectChangeSerializer'
csv_headers = [
'time', 'user', 'user_name', 'request_id', 'action', 'changed_object_type', 'changed_object_id',
'related_object_type', 'related_object_id', 'object_repr', 'object_data',

147
netbox/extras/webhooks.py
View File

@ -1,119 +1,54 @@
import time
import datetime
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
from django.db.models.signals import post_save, post_delete
from django.dispatch import Signal
from extras.models import Webhook
from utilities.utils import dynamic_import
from extras.constants import OBJECTCHANGE_ACTION_CREATE, OBJECTCHANGE_ACTION_DELETE, OBJECTCHANGE_ACTION_UPDATE
from utilities.api import get_serializer_for_model
def enqueue_webhooks(webhooks, model_class, data, event, signal_received_timestamp):
def enqueue_webhooks(instance, action):
"""
Serialize data and enqueue webhooks
Find Webhook(s) assigned to this instance + action and enqueue them
to be processed
"""
serializer_context = {
'request': None,
}
type_create = action == OBJECTCHANGE_ACTION_CREATE
type_update = action == OBJECTCHANGE_ACTION_UPDATE
type_delete = action == OBJECTCHANGE_ACTION_DELETE
if isinstance(data, list):
serializer_property = data[0].serializer
serializer_cls = dynamic_import(serializer_property)
serialized_data = serializer_cls(data, context=serializer_context, many=True)
else:
serializer_property = data.serializer
serializer_cls = dynamic_import(serializer_property)
serialized_data = serializer_cls(data, context=serializer_context)
# Find assigned webhooks
obj_type = ContentType.objects.get_for_model(instance.__class__)
webhooks = Webhook.objects.filter(
Q(enabled=True) &
(
Q(type_create=type_create) |
Q(type_update=type_update) |
Q(type_delete=type_delete)
) &
Q(obj_type=obj_type)
)
from django_rq import get_queue
webhook_queue = get_queue('default')
if webhooks:
# Get the Model's API serializer class and serialize the object
serializer_class = get_serializer_for_model(instance.__class__)
serializer_context = {
'request': None,
}
serializer = serializer_class(instance, context=serializer_context)
for webhook in webhooks:
webhook_queue.enqueue("extras.webhooks_worker.process_webhook",
webhook,
serialized_data.data,
model_class,
event,
signal_received_timestamp)
# We must only import django_rq if the Webhooks feature is enabled.
# Only if we have gotten to ths point, is the feature enabled
from django_rq import get_queue
webhook_queue = get_queue('default')
def post_save_receiver(sender, instance, created, **kwargs):
"""
Receives post_save signals from registered models. If the webhook
backend is enabled, queue any webhooks that apply to the event.
"""
if settings.WEBHOOKS_ENABLED:
signal_received_timestamp = time.time()
# look for any webhooks that match this event
updated = not created
obj_type = ContentType.objects.get_for_model(sender)
webhooks = Webhook.objects.filter(
Q(enabled=True) &
(
Q(type_create=created) |
Q(type_update=updated)
) &
Q(obj_type=obj_type)
)
event = 'created' if created else 'updated'
if webhooks:
enqueue_webhooks(webhooks, sender, instance, event, signal_received_timestamp)
def post_delete_receiver(sender, instance, **kwargs):
"""
Receives post_delete signals from registered models. If the webhook
backend is enabled, queue any webhooks that apply to the event.
"""
if settings.WEBHOOKS_ENABLED:
signal_received_timestamp = time.time()
obj_type = ContentType.objects.get_for_model(sender)
# look for any webhooks that match this event
webhooks = Webhook.objects.filter(enabled=True, type_delete=True, obj_type=obj_type)
if webhooks:
enqueue_webhooks(webhooks, sender, instance, 'deleted', signal_received_timestamp)
def bulk_operation_receiver(sender, **kwargs):
"""
Receives bulk_operation_signal signals from registered models. If the webhook
backend is enabled, queue any webhooks that apply to the event.
"""
if settings.WEBHOOKS_ENABLED:
signal_received_timestamp = time.time()
event = kwargs['event']
obj_type = ContentType.objects.get_for_model(sender)
# look for any webhooks that match this event
if event == 'created':
webhooks = Webhook.objects.filter(enabled=True, type_create=True, obj_type=obj_type)
elif event == 'updated':
webhooks = Webhook.objects.filter(enabled=True, type_update=True, obj_type=obj_type)
elif event == 'deleted':
webhooks = Webhook.objects.filter(enabled=True, type_delete=True, obj_type=obj_type)
else:
webhooks = None
if webhooks:
enqueue_webhooks(webhooks, sender, list(kwargs['instances']), event, signal_received_timestamp)
# the bulk operation signal is used to overcome signals not being sent for bulk model changes
bulk_operation_signal = Signal(providing_args=["instances", "event"])
bulk_operation_signal.connect(bulk_operation_receiver)
def register_signals(senders):
"""
Take a list of senders (Models) and register them to the post_save
and post_delete signal receivers.
"""
if settings.WEBHOOKS_ENABLED:
# only register signals if the backend is enabled
# this reduces load by not firing signals if the
# webhook backend feature is disabled
for sender in senders:
post_save.connect(post_save_receiver, sender=sender)
post_delete.connect(post_delete_receiver, sender=sender)
# enqueue the webhooks:
for webhook in webhooks:
webhook_queue.enqueue(
"extras.webhooks_worker.process_webhook",
webhook,
serializer.data,
instance.__class__,
action,
str(datetime.datetime.now())
)

4
netbox/extras/webhooks_worker.py
View File

@ -4,7 +4,7 @@ import hmac
import requests
from django_rq import job
from extras.constants import WEBHOOK_CT_JSON, WEBHOOK_CT_X_WWW_FORM_ENCODED
from extras.constants import WEBHOOK_CT_JSON, WEBHOOK_CT_X_WWW_FORM_ENCODED, OBJECTCHANGE_ACTION_CHOICES
@job('default')
@ -13,7 +13,7 @@ def process_webhook(webhook, data, model_class, event, timestamp):
Make a POST request to the defined Webhook
"""
payload = {
'event': event,
'event': dict(OBJECTCHANGE_ACTION_CHOICES)[event],
'timestamp': timestamp,
'model': model_class.__name__,
'data': data

7
netbox/ipam/apps.py
View File

@ -6,10 +6,3 @@ from django.apps import AppConfig
class IPAMConfig(AppConfig):
name = "ipam"
verbose_name = "IPAM"
def ready(self):
# register webhook signals
from extras.webhooks import register_signals
from .models import Aggregate, Prefix, IPAddress, VLAN, VRF, VLANGroup, Service
register_signals([Aggregate, Prefix, IPAddress, VLAN, VRF, VLANGroup, Service])

9
netbox/ipam/models.py
View File

@ -59,7 +59,6 @@ class VRF(ChangeLoggedModel, CustomFieldModel):
tags = TaggableManager()
serializer = 'ipam.api.serializers.VRFSerializer'
csv_headers = ['name', 'rd', 'tenant', 'enforce_unique', 'description']
class Meta:
@ -108,7 +107,6 @@ class RIR(ChangeLoggedModel):
help_text='IP space managed by this RIR is considered private'
)
serializer = 'ipam.api.serializers.RIRSerializer'
csv_headers = ['name', 'slug', 'is_private']
class Meta:
@ -162,7 +160,6 @@ class Aggregate(ChangeLoggedModel, CustomFieldModel):
tags = TaggableManager()
serializer = 'ipam.api.serializers.AggregateSerializer'
csv_headers = ['prefix', 'rir', 'date_added', 'description']
class Meta:
@ -243,7 +240,6 @@ class Role(ChangeLoggedModel):
default=1000
)
serializer = 'ipam.api.serializers.RoleSerializer'
csv_headers = ['name', 'slug', 'weight']
class Meta:
@ -336,7 +332,6 @@ class Prefix(ChangeLoggedModel, CustomFieldModel):
objects = PrefixQuerySet.as_manager()
tags = TaggableManager()
serializer = 'ipam.api.serializers.PrefixSerializer'
csv_headers = [
'prefix', 'vrf', 'tenant', 'site', 'vlan_group', 'vlan_vid', 'status', 'role', 'is_pool', 'description',
]
@ -577,7 +572,6 @@ class IPAddress(ChangeLoggedModel, CustomFieldModel):
objects = IPAddressManager()
tags = TaggableManager()
serializer = 'ipam.api.serializers.IPAddressSerializer'
csv_headers = [
'address', 'vrf', 'tenant', 'status', 'role', 'device', 'virtual_machine', 'interface_name', 'is_primary',
'description',
@ -677,7 +671,6 @@ class VLANGroup(ChangeLoggedModel):
null=True
)
serializer = 'ipam.api.serializers.VLANGroupSerializer'
csv_headers = ['name', 'slug', 'site']
class Meta:
@ -775,7 +768,6 @@ class VLAN(ChangeLoggedModel, CustomFieldModel):
tags = TaggableManager()
serializer = 'ipam.api.serializers.VLANSerializer'
csv_headers = ['site', 'group_name', 'vid', 'name', 'tenant', 'status', 'role', 'description']
class Meta:
@ -879,7 +871,6 @@ class Service(ChangeLoggedModel, CustomFieldModel):
tags = TaggableManager()
serializer = 'ipam.api.serializers.ServiceSerializer'
csv_headers = ['device', 'virtual_machine', 'name', 'protocol', 'description']
class Meta:

2
netbox/netbox/settings.py
View File

@ -175,7 +175,7 @@ MIDDLEWARE = (
'utilities.middleware.ExceptionHandlingMiddleware',
'utilities.middleware.LoginRequiredMiddleware',
'utilities.middleware.APIVersionMiddleware',
'extras.middleware.ChangeLoggingMiddleware',
'extras.middleware.ObjectChangeMiddleware',
)
ROOT_URLCONF = 'netbox.urls'

2
netbox/secrets/models.py
View File

@ -285,7 +285,6 @@ class SecretRole(ChangeLoggedModel):
blank=True
)
serializer = 'ipam.api.secrets.SecretSerializer'
csv_headers = ['name', 'slug']
class Meta:
@ -354,7 +353,6 @@ class Secret(ChangeLoggedModel, CustomFieldModel):
tags = TaggableManager()
plaintext = None
serializer = 'ipam.api.secrets.SecretSerializer'
csv_headers = ['device', 'role', 'name', 'plaintext']
class Meta:

7
netbox/tenancy/apps.py
View File

@ -5,10 +5,3 @@ from django.apps import AppConfig
class TenancyConfig(AppConfig):
name = 'tenancy'
def ready(self):
# register webhook signals
from extras.webhooks import register_signals
from .models import Tenant, TenantGroup
register_signals([Tenant, TenantGroup])

2
netbox/tenancy/models.py
View File

@ -23,7 +23,6 @@ class TenantGroup(ChangeLoggedModel):
unique=True
)
serializer = 'tenancy.api.serializers.TenantGroupSerializer'
csv_headers = ['name', 'slug']
class Meta:
@ -78,7 +77,6 @@ class Tenant(ChangeLoggedModel, CustomFieldModel):
tags = TaggableManager()
serializer = 'tenancy.api.serializers.TenantSerializer'
csv_headers = ['name', 'slug', 'group', 'description', 'comments']
class Meta:

7
netbox/utilities/views.py
View File

@ -25,7 +25,6 @@ from django.views.generic import View
from django_tables2 import RequestConfig
from extras.models import CustomField, CustomFieldValue, ExportTemplate
from extras.webhooks import bulk_operation_signal
from utilities.utils import queryset_to_csv
from utilities.forms import BootstrapMixin, CSVDataField
from .constants import M2M_FIELD_TYPES
@ -757,9 +756,6 @@ class ComponentCreateView(View):
field_links.append(field_link)
getattr(self.model, field).through.objects.bulk_create(field_links)
# send the bulk operations signal for webhooks
bulk_operation_signal.send(sender=self.model, instances=new_components, event="created")
messages.success(request, "Added {} {} to {}.".format(
len(new_components), self.model._meta.verbose_name_plural, parent
))
@ -829,9 +825,6 @@ class BulkComponentCreateView(GetReturnURLMixin, View):
if not form.errors:
self.model.objects.bulk_create(new_components)
# send the bulk operations signal for webhooks
bulk_operation_signal.send(sender=self.model, instances=new_components, event="created")
messages.success(request, "Added {} {} to {} {}.".format(
len(new_components),
self.model._meta.verbose_name_plural,

7
netbox/virtualization/apps.py
View File

@ -5,10 +5,3 @@ from django.apps import AppConfig
class VirtualizationConfig(AppConfig):
name = 'virtualization'
def ready(self):
# register webhook signals
from extras.webhooks import register_signals
from .models import Cluster, ClusterGroup, VirtualMachine
register_signals([Cluster, VirtualMachine, ClusterGroup])

4
netbox/virtualization/models.py
View File

@ -31,7 +31,6 @@ class ClusterType(ChangeLoggedModel):
unique=True
)
serializer = 'virtualization.api.serializers.ClusterTypeSerializer'
csv_headers = ['name', 'slug']
class Meta:
@ -67,7 +66,6 @@ class ClusterGroup(ChangeLoggedModel):
unique=True
)
serializer = 'virtualization.api.serializers.ClusterGroupSerializer'
csv_headers = ['name', 'slug']
class Meta:
@ -129,7 +127,6 @@ class Cluster(ChangeLoggedModel, CustomFieldModel):
tags = TaggableManager()
serializer = 'virtualization.api.serializers.ClusterSerializer'
csv_headers = ['name', 'type', 'group', 'site', 'comments']
class Meta:
@ -250,7 +247,6 @@ class VirtualMachine(ChangeLoggedModel, ConfigContextModel, CustomFieldModel):
tags = TaggableManager()
serializer = 'virtualization.api.serializers.VirtualMachineSerializer'
csv_headers = [
'name', 'status', 'role', 'cluster', 'tenant', 'platform', 'vcpus', 'memory', 'disk', 'comments',
]