From fbf91dda7d466d6d962f0a6c08001b3765029650 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 26 Oct 2021 13:41:56 -0400 Subject: [PATCH] Optimize config queries --- netbox/dcim/api/views.py | 5 ++-- netbox/dcim/models/racks.py | 4 +-- netbox/ipam/api/mixins.py | 5 ++-- netbox/ipam/models/ip.py | 6 ++--- netbox/netbox/api/pagination.py | 7 +++--- netbox/netbox/config/__init__.py | 32 ++++++++++++++++++++++-- netbox/netbox/context_processors.py | 4 +-- netbox/netbox/middleware.py | 22 +++++++++++++--- netbox/netbox/settings.py | 1 + netbox/users/views.py | 4 +-- netbox/utilities/paginator.py | 8 +++--- netbox/utilities/templatetags/helpers.py | 4 +-- netbox/utilities/tests/test_api.py | 4 +-- netbox/utilities/validators.py | 4 +-- netbox/virtualization/models.py | 4 +-- 15 files changed, 77 insertions(+), 37 deletions(-) diff --git a/netbox/dcim/api/views.py b/netbox/dcim/api/views.py index 54712a9b3..3248891f2 100644 --- a/netbox/dcim/api/views.py +++ b/netbox/dcim/api/views.py @@ -1,7 +1,6 @@ import socket from collections import OrderedDict -from django.conf import settings from django.http import Http404, HttpResponse, HttpResponseForbidden from django.shortcuts import get_object_or_404 from drf_yasg import openapi @@ -21,7 +20,7 @@ from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired from netbox.api.exceptions import ServiceUnavailable from netbox.api.metadata import ContentTypeMetadata from netbox.api.views import ModelViewSet -from netbox.config import Config +from netbox.config import get_config from utilities.api import get_serializer_for_model from utilities.utils import count_related, decode_dict from virtualization.models import VirtualMachine @@ -459,7 +458,7 @@ class DeviceViewSet(ConfigContextQuerySetMixin, CustomFieldModelViewSet): napalm_methods = request.GET.getlist('method') response = OrderedDict([(m, None) for m in napalm_methods]) - config = Config() + config = get_config() username = config.NAPALM_USERNAME password = config.NAPALM_PASSWORD timeout = config.NAPALM_TIMEOUT diff --git a/netbox/dcim/models/racks.py b/netbox/dcim/models/racks.py index 4a023477f..0bc28acaa 100644 --- a/netbox/dcim/models/racks.py +++ b/netbox/dcim/models/racks.py @@ -14,7 +14,7 @@ from dcim.choices import * from dcim.constants import * from dcim.svg import RackElevationSVG from extras.utils import extras_features -from netbox.config import Config +from netbox.config import get_config from netbox.models import OrganizationalModel, PrimaryModel from utilities.choices import ColorChoices from utilities.fields import ColorField, NaturalOrderingField @@ -394,7 +394,7 @@ class Rack(PrimaryModel): """ elevation = RackElevationSVG(self, user=user, include_images=include_images, base_url=base_url) if unit_width is None or unit_height is None: - config = Config() + config = get_config() unit_width = unit_width or config.RACK_ELEVATION_DEFAULT_UNIT_WIDTH unit_height = unit_height or config.RACK_ELEVATION_DEFAULT_UNIT_HEIGHT diff --git a/netbox/ipam/api/mixins.py b/netbox/ipam/api/mixins.py index 43d743a63..552c77d57 100644 --- a/netbox/ipam/api/mixins.py +++ b/netbox/ipam/api/mixins.py @@ -1,4 +1,3 @@ -from django.conf import settings from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.db import transaction from django.shortcuts import get_object_or_404 @@ -9,7 +8,7 @@ from rest_framework.decorators import action from rest_framework.response import Response from ipam.models import * -from netbox.config import Config +from netbox.config import get_config from utilities.constants import ADVISORY_LOCK_KEYS from . import serializers @@ -161,7 +160,7 @@ class AvailableIPsMixin: # Determine the maximum number of IPs to return else: - config = Config() + config = get_config() PAGINATE_COUNT = config.PAGINATE_COUNT MAX_PAGE_SIZE = config.MAX_PAGE_SIZE try: diff --git a/netbox/ipam/models/ip.py b/netbox/ipam/models/ip.py index af114537a..6a26f08c3 100644 --- a/netbox/ipam/models/ip.py +++ b/netbox/ipam/models/ip.py @@ -16,7 +16,7 @@ from ipam.fields import IPNetworkField, IPAddressField from ipam.managers import IPAddressManager from ipam.querysets import PrefixQuerySet from ipam.validators import DNSValidator -from netbox.config import Config +from netbox.config import get_config from utilities.querysets import RestrictedQuerySet from virtualization.models import VirtualMachine @@ -316,7 +316,7 @@ class Prefix(PrimaryModel): }) # Enforce unique IP space (if applicable) - if (self.vrf is None and Config().ENFORCE_GLOBAL_UNIQUE) or (self.vrf and self.vrf.enforce_unique): + if (self.vrf is None and get_config().ENFORCE_GLOBAL_UNIQUE) or (self.vrf and self.vrf.enforce_unique): duplicate_prefixes = self.get_duplicates() if duplicate_prefixes: raise ValidationError({ @@ -811,7 +811,7 @@ class IPAddress(PrimaryModel): }) # Enforce unique IP space (if applicable) - if (self.vrf is None and Config().ENFORCE_GLOBAL_UNIQUE) or (self.vrf and self.vrf.enforce_unique): + if (self.vrf is None and get_config().ENFORCE_GLOBAL_UNIQUE) or (self.vrf and self.vrf.enforce_unique): duplicate_ips = self.get_duplicates() if duplicate_ips and ( self.role not in IPADDRESS_ROLES_NONUNIQUE or diff --git a/netbox/netbox/api/pagination.py b/netbox/netbox/api/pagination.py index a57b0bd33..d89e32124 100644 --- a/netbox/netbox/api/pagination.py +++ b/netbox/netbox/api/pagination.py @@ -1,8 +1,7 @@ -from django.conf import settings from django.db.models import QuerySet from rest_framework.pagination import LimitOffsetPagination -from netbox.config import Config +from netbox.config import get_config class OptionalLimitOffsetPagination(LimitOffsetPagination): @@ -12,7 +11,7 @@ class OptionalLimitOffsetPagination(LimitOffsetPagination): MAX_PAGE_SIZE has been set to 0 or None. """ def __init__(self): - self.default_limit = Config().PAGINATE_COUNT + self.default_limit = get_config().PAGINATE_COUNT def paginate_queryset(self, queryset, request, view=None): @@ -44,7 +43,7 @@ class OptionalLimitOffsetPagination(LimitOffsetPagination): if limit < 0: raise ValueError() # Enforce maximum page size, if defined - MAX_PAGE_SIZE = Config().MAX_PAGE_SIZE + MAX_PAGE_SIZE = get_config().MAX_PAGE_SIZE if MAX_PAGE_SIZE: return MAX_PAGE_SIZE if limit == 0 else min(limit, MAX_PAGE_SIZE) return limit diff --git a/netbox/netbox/config/__init__.py b/netbox/netbox/config/__init__.py index 7e57f3e8d..3786c5dc8 100644 --- a/netbox/netbox/config/__init__.py +++ b/netbox/netbox/config/__init__.py @@ -1,14 +1,41 @@ +import logging +import threading + from django.conf import settings from django.core.cache import cache from .parameters import PARAMS __all__ = ( - 'Config', + 'clear_config', 'ConfigItem', + 'get_config', 'PARAMS', ) +_thread_locals = threading.local() + +logger = logging.getLogger('netbox.config') + + +def get_config(): + """ + Return the current NetBox configuration, pulling it from cache if not already loaded in memory. + """ + if not hasattr(_thread_locals, 'config'): + _thread_locals.config = Config() + logger.debug("Initialized configuration") + return _thread_locals.config + + +def clear_config(): + """ + Delete the currently loaded configuration, if any. + """ + if hasattr(_thread_locals, 'config'): + del _thread_locals.config + logger.debug("Cleared configuration") + class Config: """ @@ -19,6 +46,7 @@ class Config: self.config = cache.get('config') self.version = cache.get('config_version') self.defaults = {param.name: param.default for param in PARAMS} + logger.debug("Loaded configuration data from cache") def __getattr__(self, item): @@ -46,5 +74,5 @@ class ConfigItem: self.item = item def __call__(self): - config = Config() + config = get_config() return getattr(config, self.item) diff --git a/netbox/netbox/context_processors.py b/netbox/netbox/context_processors.py index 8ae0a0f26..74178ceb4 100644 --- a/netbox/netbox/context_processors.py +++ b/netbox/netbox/context_processors.py @@ -1,7 +1,7 @@ from django.conf import settings as django_settings from extras.registry import registry -from netbox.config import Config +from netbox.config import get_config def settings_and_registry(request): @@ -10,7 +10,7 @@ def settings_and_registry(request): """ return { 'settings': django_settings, - 'config': Config(), + 'config': get_config(), 'registry': registry, 'preferences': request.user.config if request.user.is_authenticated else {}, } diff --git a/netbox/netbox/middleware.py b/netbox/netbox/middleware.py index a8f989a2a..8d03c6aee 100644 --- a/netbox/netbox/middleware.py +++ b/netbox/netbox/middleware.py @@ -11,11 +11,12 @@ from django.http import Http404, HttpResponseRedirect from django.urls import reverse from extras.context_managers import change_logging +from netbox.config import clear_config from netbox.views import server_error from utilities.api import is_api_request, rest_api_server_error -class LoginRequiredMiddleware(object): +class LoginRequiredMiddleware: """ If LOGIN_REQUIRED is True, redirect all non-authenticated users to the login page. """ @@ -114,7 +115,7 @@ class RemoteUserMiddleware(RemoteUserMiddleware_): return groups -class ObjectChangeMiddleware(object): +class ObjectChangeMiddleware: """ This middleware performs three functions in response to an object being created, updated, or deleted: @@ -144,7 +145,7 @@ class ObjectChangeMiddleware(object): return response -class APIVersionMiddleware(object): +class APIVersionMiddleware: """ If the request is for an API endpoint, include the API version as a response header. """ @@ -159,7 +160,20 @@ class APIVersionMiddleware(object): return response -class ExceptionHandlingMiddleware(object): +class DynamicConfigMiddleware: + """ + Store the cached NetBox configuration in thread-local storage for the duration of the request. + """ + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + response = self.get_response(request) + clear_config() + return response + + +class ExceptionHandlingMiddleware: """ Intercept certain exceptions which are likely indicative of installation issues and provide helpful instructions to the user. diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 1ff000ff8..45475ef9a 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -335,6 +335,7 @@ MIDDLEWARE = [ 'netbox.middleware.ExceptionHandlingMiddleware', 'netbox.middleware.RemoteUserMiddleware', 'netbox.middleware.LoginRequiredMiddleware', + 'netbox.middleware.DynamicConfigMiddleware', 'netbox.middleware.APIVersionMiddleware', 'netbox.middleware.ObjectChangeMiddleware', 'django_prometheus.middleware.PrometheusAfterMiddleware', diff --git a/netbox/users/views.py b/netbox/users/views.py index dab0d4db0..ab17955e3 100644 --- a/netbox/users/views.py +++ b/netbox/users/views.py @@ -13,7 +13,7 @@ from django.utils.http import is_safe_url from django.views.decorators.debug import sensitive_post_parameters from django.views.generic import View -from netbox.config import Config +from netbox.config import get_config from utilities.forms import ConfirmationForm from .forms import LoginForm, PasswordChangeForm, TokenForm from .models import Token @@ -53,7 +53,7 @@ class LoginView(View): # If maintenance mode is enabled, assume the database is read-only, and disable updating the user's # last_login time upon authentication. - if Config().MAINTENANCE_MODE: + if get_config().MAINTENANCE_MODE: logger.warning("Maintenance mode enabled: disabling update of most recent login time") user_logged_in.disconnect(update_last_login, dispatch_uid='update_last_login') diff --git a/netbox/utilities/paginator.py b/netbox/utilities/paginator.py index 20f3f6de2..4cc3ef601 100644 --- a/netbox/utilities/paginator.py +++ b/netbox/utilities/paginator.py @@ -1,6 +1,6 @@ from django.core.paginator import Paginator, Page -from netbox.config import Config +from netbox.config import get_config class EnhancedPaginator(Paginator): @@ -14,9 +14,9 @@ class EnhancedPaginator(Paginator): try: per_page = int(per_page) if per_page < 1: - per_page = Config().PAGINATE_COUNT + per_page = get_config().PAGINATE_COUNT except ValueError: - per_page = Config().PAGINATE_COUNT + per_page = get_config().PAGINATE_COUNT # Set orphans count based on page size if orphans is None and per_page <= 50: @@ -66,7 +66,7 @@ def get_paginate_count(request): Return the lesser of the calculated value and MAX_PAGE_SIZE. """ - config = Config() + config = get_config() if 'per_page' in request.GET: try: diff --git a/netbox/utilities/templatetags/helpers.py b/netbox/utilities/templatetags/helpers.py index 833d19535..9b510d9ed 100644 --- a/netbox/utilities/templatetags/helpers.py +++ b/netbox/utilities/templatetags/helpers.py @@ -14,7 +14,7 @@ from django.utils.html import strip_tags from django.utils.safestring import mark_safe from markdown import markdown -from netbox.config import Config +from netbox.config import get_config from utilities.forms import get_selected_values, TableConfigForm from utilities.utils import foreground_color @@ -45,7 +45,7 @@ def render_markdown(value): value = strip_tags(value) # Sanitize Markdown links - schemes = '|'.join(Config().ALLOWED_URL_SCHEMES) + schemes = '|'.join(get_config().ALLOWED_URL_SCHEMES) pattern = fr'\[(.+)\]\((?!({schemes})).*:(.+)\)' value = re.sub(pattern, '[\\1](\\3)', value, flags=re.IGNORECASE) diff --git a/netbox/utilities/tests/test_api.py b/netbox/utilities/tests/test_api.py index cc3bb1ddc..1171bd496 100644 --- a/netbox/utilities/tests/test_api.py +++ b/netbox/utilities/tests/test_api.py @@ -9,7 +9,7 @@ from dcim.models import Region, Site from extras.choices import CustomFieldTypeChoices from extras.models import CustomField from ipam.models import VLAN -from netbox.config import Config +from netbox.config import get_config from utilities.testing import APITestCase, disable_warnings @@ -137,7 +137,7 @@ class APIPaginationTestCase(APITestCase): def test_default_page_size(self): response = self.client.get(self.url, format='json', **self.header) - page_size = Config().PAGINATE_COUNT + page_size = get_config().PAGINATE_COUNT self.assertLess(page_size, 100, "Default page size not sufficient for data set") self.assertHttpStatus(response, status.HTTP_200_OK) diff --git a/netbox/utilities/validators.py b/netbox/utilities/validators.py index 5b5775482..5fce17a3a 100644 --- a/netbox/utilities/validators.py +++ b/netbox/utilities/validators.py @@ -3,7 +3,7 @@ import re from django.core.exceptions import ValidationError from django.core.validators import _lazy_re_compile, BaseValidator, URLValidator -from netbox.config import Config +from netbox.config import get_config class EnhancedURLValidator(URLValidator): @@ -24,7 +24,7 @@ class EnhancedURLValidator(URLValidator): def __init__(self, schemes=None, **kwargs): super().__init__(**kwargs) if schemes is not None: - self.schemes = Config().ALLOWED_URL_SCHEMES + self.schemes = get_config().ALLOWED_URL_SCHEMES class ExclusionValidator(BaseValidator): diff --git a/netbox/virtualization/models.py b/netbox/virtualization/models.py index f82550b4f..db2404546 100644 --- a/netbox/virtualization/models.py +++ b/netbox/virtualization/models.py @@ -8,7 +8,7 @@ from dcim.models import BaseInterface, Device from extras.models import ConfigContextModel from extras.querysets import ConfigContextModelQuerySet from extras.utils import extras_features -from netbox.config import Config +from netbox.config import get_config from netbox.models import OrganizationalModel, PrimaryModel from utilities.fields import NaturalOrderingField from utilities.ordering import naturalize_interface @@ -340,7 +340,7 @@ class VirtualMachine(PrimaryModel, ConfigContextModel): @property def primary_ip(self): - if Config().PREFER_IPV4 and self.primary_ip4: + if get_config().PREFER_IPV4 and self.primary_ip4: return self.primary_ip4 elif self.primary_ip6: return self.primary_ip6