Merge branch 'develop' into feature

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -14,7 +14,7 @@ body:
     attributes:
       label: NetBox version
       description: What version of NetBox are you currently running?
-      placeholder: v3.0.10
+      placeholder: v3.0.11
     validations:
       required: true
   - type: dropdown

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -14,7 +14,7 @@ body:
     attributes:
       label: NetBox version
       description: What version of NetBox are you currently running?
-      placeholder: v3.0.10
+      placeholder: v3.0.11
     validations:
       required: true
   - type: dropdown

docs/release-notes/version-3.0.md

@@ -1,6 +1,10 @@
 # NetBox v3.0
 
-## v3.0.11 (FUTURE)
+## v3.0.12 (FUTURE)
+
+---
+
+## v3.0.11 (2021-11-24)
 
 ### Enhancements
 
@@ -14,6 +18,7 @@
 ### Bug Fixes
 
 * [#7399](https://github.com/netbox-community/netbox/issues/7399) - Fix excessive CPU utilization when `AUTH_LDAP_FIND_GROUP_PERMS` is enabled
+* [#7657](https://github.com/netbox-community/netbox/issues/7657) - Make change logging middleware thread-safe
 * [#7720](https://github.com/netbox-community/netbox/issues/7720) - Fix initialization of custom script MultiObjectVar field with multiple values
 * [#7729](https://github.com/netbox-community/netbox/issues/7729) - Fix permissions evaluation when displaying VLAN group VLANs table
 * [#7739](https://github.com/netbox-community/netbox/issues/7739) - Fix exception when tracing cable across circuit with no far end termination
@@ -448,7 +453,7 @@ Note that NetBox's `rqworker` process will _not_ service custom queues by defaul
 * [#6154](https://github.com/netbox-community/netbox/issues/6154) - Allow decimal values for cable lengths
 * [#6328](https://github.com/netbox-community/netbox/issues/6328) - Build and serve documentation locally
 
-### Bug Fixes (from v3.2-beta2)
+### Bug Fixes (from v3.0-beta2)
 
 * [#6977](https://github.com/netbox-community/netbox/issues/6977) - Truncate global search dropdown on small screens
 * [#6979](https://github.com/netbox-community/netbox/issues/6979) - Hide "create & add another" button for circuit terminations

netbox/dcim/choices.py

@@ -447,7 +447,7 @@ class PowerPortTypeChoices(ChoiceSet):
         )),
         ('International/ITA', (
             (TYPE_ITA_C, 'ITA Type C (CEE 7/16)'),
-            (TYPE_ITA_E, 'ITA Type E (CEE 7/5)'),
+            (TYPE_ITA_E, 'ITA Type E (CEE 7/6)'),
             (TYPE_ITA_F, 'ITA Type F (CEE 7/4)'),
             (TYPE_ITA_EF, 'ITA Type E/F (CEE 7/7)'),
             (TYPE_ITA_G, 'ITA Type G (BS 1363)'),
@@ -659,8 +659,8 @@ class PowerOutletTypeChoices(ChoiceSet):
             (TYPE_CS8464C, 'CS8464C'),
         )),
         ('ITA/International', (
-            (TYPE_ITA_E, 'ITA Type E (CEE7/5)'),
+            (TYPE_ITA_E, 'ITA Type E (CEE 7/5)'),
-            (TYPE_ITA_F, 'ITA Type F (CEE7/3)'),
+            (TYPE_ITA_F, 'ITA Type F (CEE 7/3)'),
             (TYPE_ITA_G, 'ITA Type G (BS 1363)'),
             (TYPE_ITA_H, 'ITA Type H'),
             (TYPE_ITA_I, 'ITA Type I'),

netbox/extras/context_managers.py

@@ -2,8 +2,9 @@ from contextlib import contextmanager
 
 from django.db.models.signals import m2m_changed, pre_delete, post_save
 
-from extras.signals import clear_webhooks, _clear_webhook_queue, _handle_changed_object, _handle_deleted_object
+from extras.signals import clear_webhooks, clear_webhook_queue, handle_changed_object, handle_deleted_object
-from utilities.utils import curry
+from netbox import thread_locals
+from netbox.request_context import set_request
 from .webhooks import flush_webhooks
 
 
@@ -15,12 +16,8 @@ def change_logging(request):
 
     :param request: WSGIRequest object with a unique `id` set
     """
-    webhook_queue = []
+    set_request(request)
-
+    thread_locals.webhook_queue = []
-    # Curry signals receivers to pass the current request
-    handle_changed_object = curry(_handle_changed_object, request, webhook_queue)
-    handle_deleted_object = curry(_handle_deleted_object, request, webhook_queue)
-    clear_webhook_queue = curry(_clear_webhook_queue, webhook_queue)
 
     # Connect our receivers to the post_save and post_delete signals.
     post_save.connect(handle_changed_object, dispatch_uid='handle_changed_object')
@@ -38,5 +35,8 @@ def change_logging(request):
     clear_webhooks.disconnect(clear_webhook_queue, dispatch_uid='clear_webhook_queue')
 
     # Flush queued webhooks to RQ
-    flush_webhooks(webhook_queue)
+    flush_webhooks(thread_locals.webhook_queue)
-    del webhook_queue
+    del thread_locals.webhook_queue
+
+    # Clear the request from thread-local storage
+    set_request(None)

netbox/extras/signals.py

@@ -7,13 +7,14 @@ from django.dispatch import receiver, Signal
 from django_prometheus.models import model_deletes, model_inserts, model_updates
 
 from extras.validators import CustomValidator
+from netbox import thread_locals
 from netbox.config import get_config
+from netbox.request_context import get_request
 from netbox.signals import post_clean
 from .choices import ObjectChangeActionChoices
 from .models import ConfigRevision, CustomField, ObjectChange
 from .webhooks import enqueue_object, get_snapshots, serialize_for_webhook
 
-
 #
 # Change logging/webhooks
 #
@@ -22,10 +23,16 @@ from .webhooks import enqueue_object, get_snapshots, serialize_for_webhook
 clear_webhooks = Signal()
 
 
-def _handle_changed_object(request, webhook_queue, sender, instance, **kwargs):
+def handle_changed_object(sender, instance, **kwargs):
     """
     Fires when an object is created or updated.
     """
+    if not hasattr(instance, 'to_objectchange'):
+        return
+
+    request = get_request()
+    m2m_changed = False
+
     def is_same_object(instance, webhook_data):
         return (
             ContentType.objects.get_for_model(instance) == webhook_data['content_type'] and
@@ -33,11 +40,6 @@ def _handle_changed_object(request, webhook_queue, sender, instance, **kwargs):
             request.id == webhook_data['request_id']
         )
 
-    if not hasattr(instance, 'to_objectchange'):
-        return
-
-    m2m_changed = False
-
     # Determine the type of change being made
     if kwargs.get('created'):
         action = ObjectChangeActionChoices.ACTION_CREATE
@@ -67,6 +69,7 @@ def _handle_changed_object(request, webhook_queue, sender, instance, **kwargs):
             objectchange.save()
 
     # If this is an M2M change, update the previously queued webhook (from post_save)
+    webhook_queue = thread_locals.webhook_queue
     if m2m_changed and webhook_queue and is_same_object(instance, webhook_queue[-1]):
         instance.refresh_from_db()  # Ensure that we're working with fresh M2M assignments
         webhook_queue[-1]['data'] = serialize_for_webhook(instance)
@@ -81,13 +84,15 @@ def _handle_changed_object(request, webhook_queue, sender, instance, **kwargs):
         model_updates.labels(instance._meta.model_name).inc()
 
 
-def _handle_deleted_object(request, webhook_queue, sender, instance, **kwargs):
+def handle_deleted_object(sender, instance, **kwargs):
     """
     Fires when an object is deleted.
     """
     if not hasattr(instance, 'to_objectchange'):
         return
 
+    request = get_request()
+
     # Record an ObjectChange if applicable
     if hasattr(instance, 'to_objectchange'):
         objectchange = instance.to_objectchange(ObjectChangeActionChoices.ACTION_DELETE)
@@ -96,19 +101,21 @@ def _handle_deleted_object(request, webhook_queue, sender, instance, **kwargs):
         objectchange.save()
 
     # Enqueue webhooks
+    webhook_queue = thread_locals.webhook_queue
     enqueue_object(webhook_queue, instance, request.user, request.id, ObjectChangeActionChoices.ACTION_DELETE)
 
     # Increment metric counters
     model_deletes.labels(instance._meta.model_name).inc()
 
 
-def _clear_webhook_queue(webhook_queue, sender, **kwargs):
+def clear_webhook_queue(sender, **kwargs):
     """
     Delete any queued webhooks (e.g. because of an aborted bulk transaction)
     """
     logger = logging.getLogger('webhooks')
-    logger.info(f"Clearing {len(webhook_queue)} queued webhooks ({sender})")
+    webhook_queue = thread_locals.webhook_queue
 
+    logger.info(f"Clearing {len(webhook_queue)} queued webhooks ({sender})")
     webhook_queue.clear()
 
 

netbox/netbox/__init__.py

@@ -0,0 +1,3 @@
+import threading
+
+thread_locals = threading.local()

netbox/netbox/middleware.py

@@ -1,10 +1,10 @@
+import logging
 import uuid
 from urllib import parse
-import logging
 
 from django.conf import settings
-from django.contrib.auth.middleware import RemoteUserMiddleware as RemoteUserMiddleware_
 from django.contrib import auth
+from django.contrib.auth.middleware import RemoteUserMiddleware as RemoteUserMiddleware_
 from django.core.exceptions import ImproperlyConfigured
 from django.db import ProgrammingError
 from django.http import Http404, HttpResponseRedirect

netbox/netbox/request_context.py

@@ -0,0 +1,9 @@
+from netbox import thread_locals
+
+
+def set_request(request):
+    thread_locals.request = request
+
+
+def get_request():
+    return getattr(thread_locals, 'request', None)

netbox/templates/extras/webhook.html

@@ -149,7 +149,11 @@
         Additional Headers
       </h5>
       <div class="card-body">
-        <pre>{{ object.additional_headers }}</pre>
+        {% if object.additional_headers %}
+          <pre>{{ object.additional_headers }}</pre>
+        {% else %}
+          <span class="text-muted">None</span>
+        {% endif %}
       </div>
     </div>
     <div class="card">
@@ -157,7 +161,11 @@
         Body Template
       </h5>
       <div class="card-body">
-        <pre>{{ object.body_template }}</pre>
+        {% if object.body_template %}
+          <pre>{{ object.body_template }}</pre>
+        {% else %}
+          <span class="text-muted">None</span>
+        {% endif %}
       </div>
     </div>
     {% plugin_right_page object %}

netbox/utilities/utils.py

@@ -327,13 +327,6 @@ def decode_dict(encoded_dict: Dict, *, decode_keys: bool = True) -> Dict:
     return {urllib.parse.unquote(k): decode_value(v, decode_keys) for k, v in encoded_dict.items()}
 
 
-# Taken from django.utils.functional (<3.0)
-def curry(_curried_func, *args, **kwargs):
-    def _curried(*moreargs, **morekwargs):
-        return _curried_func(*args, *moreargs, **{**kwargs, **morekwargs})
-    return _curried
-
-
 def array_to_string(array):
     """
     Generate an efficient, human-friendly string from a set of integers. Intended for use with ArrayField.

requirements.txt

@@ -7,7 +7,7 @@ django-mptt==0.13.4
 django-pglocks==1.0.4
 django-prometheus==2.1.0
 django-redis==5.0.0
-django-rq==2.4.1
+django-rq==2.5.1
 django-tables2==2.4.1
 django-taggit==1.5.1
 django-timezone-field==4.2.1
@@ -16,7 +16,7 @@ drf-yasg[validation]==1.20.0
 graphene_django==2.15.0
 gunicorn==20.1.0
 Jinja2==3.0.3
-Markdown==3.3.4
+Markdown==3.3.6
 markdown-include==0.6.0
 mkdocs-material==7.3.6
 netaddr==0.8.0