From 0d600995883ce3ec5758f850d5fc7fc2f823e0d1 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 17 Nov 2021 14:26:12 -0500 Subject: [PATCH 1/8] Move request object and webhook queue to TLS --- netbox/extras/context_managers.py | 20 ++++++++++---------- netbox/extras/signals.py | 26 +++++++++++++++++--------- netbox/netbox/__init__.py | 3 +++ netbox/netbox/middleware.py | 6 +++--- netbox/netbox/request_context.py | 9 +++++++++ netbox/utilities/utils.py | 7 ------- 6 files changed, 42 insertions(+), 29 deletions(-) create mode 100644 netbox/netbox/request_context.py diff --git a/netbox/extras/context_managers.py b/netbox/extras/context_managers.py index 66b5ff94d..9f73fe9c3 100644 --- a/netbox/extras/context_managers.py +++ b/netbox/extras/context_managers.py @@ -2,8 +2,9 @@ from contextlib import contextmanager from django.db.models.signals import m2m_changed, pre_delete, post_save -from extras.signals import clear_webhooks, _clear_webhook_queue, _handle_changed_object, _handle_deleted_object -from utilities.utils import curry +from extras.signals import clear_webhooks, clear_webhook_queue, handle_changed_object, handle_deleted_object +from netbox import thread_locals +from netbox.request_context import set_request from .webhooks import flush_webhooks @@ -15,12 +16,8 @@ def change_logging(request): :param request: WSGIRequest object with a unique `id` set """ - webhook_queue = [] - - # Curry signals receivers to pass the current request - handle_changed_object = curry(_handle_changed_object, request, webhook_queue) - handle_deleted_object = curry(_handle_deleted_object, request, webhook_queue) - clear_webhook_queue = curry(_clear_webhook_queue, webhook_queue) + set_request(request) + thread_locals.webhook_queue = [] # Connect our receivers to the post_save and post_delete signals. post_save.connect(handle_changed_object, dispatch_uid='handle_changed_object') @@ -38,5 +35,8 @@ def change_logging(request): clear_webhooks.disconnect(clear_webhook_queue, dispatch_uid='clear_webhook_queue') # Flush queued webhooks to RQ - flush_webhooks(webhook_queue) - del webhook_queue + flush_webhooks(thread_locals.webhook_queue) + del thread_locals.webhook_queue + + # Clear the request from thread-local storage + set_request(None) diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py index 4f09706be..ec3653e15 100644 --- a/netbox/extras/signals.py +++ b/netbox/extras/signals.py @@ -6,6 +6,8 @@ from django.db.models.signals import m2m_changed, post_save, pre_delete from django.dispatch import receiver, Signal from django_prometheus.models import model_deletes, model_inserts, model_updates +from netbox import thread_locals +from netbox.request_context import get_request from netbox.signals import post_clean from .choices import ObjectChangeActionChoices from .models import CustomField, ObjectChange @@ -20,10 +22,16 @@ from .webhooks import enqueue_object, get_snapshots, serialize_for_webhook clear_webhooks = Signal() -def _handle_changed_object(request, webhook_queue, sender, instance, **kwargs): +def handle_changed_object(sender, instance, **kwargs): """ Fires when an object is created or updated. """ + if not hasattr(instance, 'to_objectchange'): + return + + request = get_request() + m2m_changed = False + def is_same_object(instance, webhook_data): return ( ContentType.objects.get_for_model(instance) == webhook_data['content_type'] and @@ -31,11 +39,6 @@ def _handle_changed_object(request, webhook_queue, sender, instance, **kwargs): request.id == webhook_data['request_id'] ) - if not hasattr(instance, 'to_objectchange'): - return - - m2m_changed = False - # Determine the type of change being made if kwargs.get('created'): action = ObjectChangeActionChoices.ACTION_CREATE @@ -65,6 +68,7 @@ def _handle_changed_object(request, webhook_queue, sender, instance, **kwargs): objectchange.save() # If this is an M2M change, update the previously queued webhook (from post_save) + webhook_queue = thread_locals.webhook_queue if m2m_changed and webhook_queue and is_same_object(instance, webhook_queue[-1]): instance.refresh_from_db() # Ensure that we're working with fresh M2M assignments webhook_queue[-1]['data'] = serialize_for_webhook(instance) @@ -79,13 +83,15 @@ def _handle_changed_object(request, webhook_queue, sender, instance, **kwargs): model_updates.labels(instance._meta.model_name).inc() -def _handle_deleted_object(request, webhook_queue, sender, instance, **kwargs): +def handle_deleted_object(sender, instance, **kwargs): """ Fires when an object is deleted. """ if not hasattr(instance, 'to_objectchange'): return + request = get_request() + # Record an ObjectChange if applicable if hasattr(instance, 'to_objectchange'): objectchange = instance.to_objectchange(ObjectChangeActionChoices.ACTION_DELETE) @@ -94,19 +100,21 @@ def _handle_deleted_object(request, webhook_queue, sender, instance, **kwargs): objectchange.save() # Enqueue webhooks + webhook_queue = thread_locals.webhook_queue enqueue_object(webhook_queue, instance, request.user, request.id, ObjectChangeActionChoices.ACTION_DELETE) # Increment metric counters model_deletes.labels(instance._meta.model_name).inc() -def _clear_webhook_queue(webhook_queue, sender, **kwargs): +def clear_webhook_queue(sender, **kwargs): """ Delete any queued webhooks (e.g. because of an aborted bulk transaction) """ logger = logging.getLogger('webhooks') - logger.info(f"Clearing {len(webhook_queue)} queued webhooks ({sender})") + webhook_queue = thread_locals.webhook_queue + logger.info(f"Clearing {len(webhook_queue)} queued webhooks ({sender})") webhook_queue.clear() diff --git a/netbox/netbox/__init__.py b/netbox/netbox/__init__.py index e69de29bb..5cf431025 100644 --- a/netbox/netbox/__init__.py +++ b/netbox/netbox/__init__.py @@ -0,0 +1,3 @@ +import threading + +thread_locals = threading.local() diff --git a/netbox/netbox/middleware.py b/netbox/netbox/middleware.py index a8f989a2a..ed308ea54 100644 --- a/netbox/netbox/middleware.py +++ b/netbox/netbox/middleware.py @@ -1,10 +1,10 @@ +import logging import uuid from urllib import parse -import logging from django.conf import settings -from django.contrib.auth.middleware import RemoteUserMiddleware as RemoteUserMiddleware_ from django.contrib import auth +from django.contrib.auth.middleware import RemoteUserMiddleware as RemoteUserMiddleware_ from django.core.exceptions import ImproperlyConfigured from django.db import ProgrammingError from django.http import Http404, HttpResponseRedirect @@ -114,7 +114,7 @@ class RemoteUserMiddleware(RemoteUserMiddleware_): return groups -class ObjectChangeMiddleware(object): +class ObjectChangeMiddleware: """ This middleware performs three functions in response to an object being created, updated, or deleted: diff --git a/netbox/netbox/request_context.py b/netbox/netbox/request_context.py new file mode 100644 index 000000000..41e8283e8 --- /dev/null +++ b/netbox/netbox/request_context.py @@ -0,0 +1,9 @@ +from netbox import thread_locals + + +def set_request(request): + thread_locals.request = request + + +def get_request(): + return getattr(thread_locals, 'request', None) diff --git a/netbox/utilities/utils.py b/netbox/utilities/utils.py index 9ddb072f3..8fab500dc 100644 --- a/netbox/utilities/utils.py +++ b/netbox/utilities/utils.py @@ -327,13 +327,6 @@ def decode_dict(encoded_dict: Dict, *, decode_keys: bool = True) -> Dict: return {urllib.parse.unquote(k): decode_value(v, decode_keys) for k, v in encoded_dict.items()} -# Taken from django.utils.functional (<3.0) -def curry(_curried_func, *args, **kwargs): - def _curried(*moreargs, **morekwargs): - return _curried_func(*args, *moreargs, **{**kwargs, **morekwargs}) - return _curried - - def array_to_string(array): """ Generate an efficient, human-friendly string from a set of integers. Intended for use with ArrayField. From eded00cbb3444a0e5de0bd08acb02c96875c7ed0 Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sun, 21 Nov 2021 22:23:29 +0100 Subject: [PATCH 2/8] chore: Always use "CEE 7" (with the space) consistently --- netbox/dcim/choices.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netbox/dcim/choices.py b/netbox/dcim/choices.py index d3cdbdf8f..40c61e899 100644 --- a/netbox/dcim/choices.py +++ b/netbox/dcim/choices.py @@ -640,8 +640,8 @@ class PowerOutletTypeChoices(ChoiceSet): (TYPE_CS8464C, 'CS8464C'), )), ('ITA/International', ( - (TYPE_ITA_E, 'ITA Type E (CEE7/5)'), - (TYPE_ITA_F, 'ITA Type F (CEE7/3)'), + (TYPE_ITA_E, 'ITA Type E (CEE 7/5)'), + (TYPE_ITA_F, 'ITA Type F (CEE 7/3)'), (TYPE_ITA_G, 'ITA Type G (BS 1363)'), (TYPE_ITA_H, 'ITA Type H'), (TYPE_ITA_I, 'ITA Type I'), From 175498940e77291768137432c9ea12b0620b22f9 Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sun, 21 Nov 2021 23:41:36 +0100 Subject: [PATCH 3/8] Fixes #7897: CEE 7/5 is only a power outlet, no power port Ref: * https://en.wikipedia.org/wiki/CEE_7_standard_AC_plugs_and_sockets#CEE_7/5_socket_and_CEE_7/6_plug_(French;_Type_E) * https://blog.packetsar.com/wp-content/uploads/Power_and_Cooling_Cheat_Sheet.pdf --- netbox/dcim/choices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/dcim/choices.py b/netbox/dcim/choices.py index 40c61e899..36eb24c96 100644 --- a/netbox/dcim/choices.py +++ b/netbox/dcim/choices.py @@ -428,7 +428,7 @@ class PowerPortTypeChoices(ChoiceSet): )), ('International/ITA', ( (TYPE_ITA_C, 'ITA Type C (CEE 7/16)'), - (TYPE_ITA_E, 'ITA Type E (CEE 7/5)'), + (TYPE_ITA_E, 'ITA Type E (CEE 7/6)'), (TYPE_ITA_F, 'ITA Type F (CEE 7/4)'), (TYPE_ITA_EF, 'ITA Type E/F (CEE 7/7)'), (TYPE_ITA_G, 'ITA Type G (BS 1363)'), From f90c591c785df5493e1fb8f1ee0e193bbe63e15a Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 22 Nov 2021 13:36:51 -0500 Subject: [PATCH 4/8] Fixes #7890: Correct typo --- docs/release-notes/version-3.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes/version-3.0.md b/docs/release-notes/version-3.0.md index 184b5debb..92a3dd39e 100644 --- a/docs/release-notes/version-3.0.md +++ b/docs/release-notes/version-3.0.md @@ -448,7 +448,7 @@ Note that NetBox's `rqworker` process will _not_ service custom queues by defaul * [#6154](https://github.com/netbox-community/netbox/issues/6154) - Allow decimal values for cable lengths * [#6328](https://github.com/netbox-community/netbox/issues/6328) - Build and serve documentation locally -### Bug Fixes (from v3.2-beta2) +### Bug Fixes (from v3.0-beta2) * [#6977](https://github.com/netbox-community/netbox/issues/6977) - Truncate global search dropdown on small screens * [#6979](https://github.com/netbox-community/netbox/issues/6979) - Hide "create & add another" button for circuit terminations From 1e42fecf6645bfdf4b42a00b94e02c136a2c6f9e Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 24 Nov 2021 09:15:30 -0500 Subject: [PATCH 5/8] Changelog for #7657 --- docs/release-notes/version-3.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-3.0.md b/docs/release-notes/version-3.0.md index 92a3dd39e..4bf7e54ce 100644 --- a/docs/release-notes/version-3.0.md +++ b/docs/release-notes/version-3.0.md @@ -14,6 +14,7 @@ ### Bug Fixes * [#7399](https://github.com/netbox-community/netbox/issues/7399) - Fix excessive CPU utilization when `AUTH_LDAP_FIND_GROUP_PERMS` is enabled +* [#7657](https://github.com/netbox-community/netbox/issues/7657) - Make change logging middleware thread-safe * [#7720](https://github.com/netbox-community/netbox/issues/7720) - Fix initialization of custom script MultiObjectVar field with multiple values * [#7729](https://github.com/netbox-community/netbox/issues/7729) - Fix permissions evaluation when displaying VLAN group VLANs table * [#7739](https://github.com/netbox-community/netbox/issues/7739) - Fix exception when tracing cable across circuit with no far end termination From 416caa8f50763d3ee6a8340f9e9fedbd53b5b4d0 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 24 Nov 2021 13:17:59 -0500 Subject: [PATCH 6/8] Hide code blocks when not needed --- netbox/templates/extras/webhook.html | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/netbox/templates/extras/webhook.html b/netbox/templates/extras/webhook.html index c92ec4c99..266fa9263 100644 --- a/netbox/templates/extras/webhook.html +++ b/netbox/templates/extras/webhook.html @@ -137,7 +137,11 @@ Additional Headers
- 
{{ object.additional_headers }}
+ {% if object.additional_headers %} + 
{{ object.additional_headers }}
+ {% else %} + None + {% endif %}
@@ -145,7 +149,11 @@ Body Template
- 
{{ object.body_template }}
+ {% if object.body_template %} + 
{{ object.body_template }}
+ {% else %} + None + {% endif %}
{% plugin_right_page object %} From 57ccbf44b8854311880499f99a9dc6b013b390be Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 24 Nov 2021 13:25:57 -0500 Subject: [PATCH 7/8] Release v3.0.11 --- .github/ISSUE_TEMPLATE/bug_report.yaml | 2 +- .github/ISSUE_TEMPLATE/feature_request.yaml | 2 +- docs/release-notes/version-3.0.md | 2 +- netbox/netbox/settings.py | 2 +- requirements.txt | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml index 79fb0e334..4a6dba734 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yaml +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.0.10 + placeholder: v3.0.11 validations: required: true - type: dropdown diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index 76944eecb..4c3ab0277 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -14,7 +14,7 @@ body: attributes: label: NetBox version description: What version of NetBox are you currently running? - placeholder: v3.0.10 + placeholder: v3.0.11 validations: required: true - type: dropdown diff --git a/docs/release-notes/version-3.0.md b/docs/release-notes/version-3.0.md index 4bf7e54ce..f76869f6e 100644 --- a/docs/release-notes/version-3.0.md +++ b/docs/release-notes/version-3.0.md @@ -1,6 +1,6 @@ # NetBox v3.0 -## v3.0.11 (FUTURE) +## v3.0.11 (2021-11-24) ### Enhancements diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index b7886b23c..8538c1d36 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -17,7 +17,7 @@ from django.core.validators import URLValidator # Environment setup # -VERSION = '3.0.11-dev' +VERSION = '3.0.11' # Hostname HOSTNAME = platform.node() diff --git a/requirements.txt b/requirements.txt index 84ad0c398..0f089a87c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ django-mptt==0.13.4 django-pglocks==1.0.4 django-prometheus==2.1.0 django-redis==5.0.0 -django-rq==2.4.1 +django-rq==2.5.1 django-tables2==2.4.1 django-taggit==1.5.1 django-timezone-field==4.2.1 @@ -16,7 +16,7 @@ drf-yasg[validation]==1.20.0 graphene_django==2.15.0 gunicorn==20.1.0 Jinja2==3.0.3 -Markdown==3.3.4 +Markdown==3.3.6 markdown-include==0.6.0 mkdocs-material==7.3.6 netaddr==0.8.0 From 86ada33577786140899c28792c8c26fc68d599c5 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Wed, 24 Nov 2021 13:58:57 -0500 Subject: [PATCH 8/8] PRVB --- docs/release-notes/version-3.0.md | 4 ++++ netbox/netbox/settings.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-3.0.md b/docs/release-notes/version-3.0.md index f76869f6e..f7fbe06ab 100644 --- a/docs/release-notes/version-3.0.md +++ b/docs/release-notes/version-3.0.md @@ -1,5 +1,9 @@ # NetBox v3.0 +## v3.0.12 (FUTURE) + +--- + ## v3.0.11 (2021-11-24) ### Enhancements diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 8538c1d36..7c205cca4 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -17,7 +17,7 @@ from django.core.validators import URLValidator # Environment setup # -VERSION = '3.0.11' +VERSION = '3.0.12-dev' # Hostname HOSTNAME = platform.node()