From 22d2289ed2b809f9e676d8966fa99aa6268b150e Mon Sep 17 00:00:00 2001 From: John Anderson Date: Fri, 23 Oct 2020 01:18:04 -0400 Subject: [PATCH] add support for regions and vms --- netbox/dcim/api/views.py | 28 +++++++----- netbox/dcim/models/devices.py | 4 +- netbox/dcim/views.py | 2 +- netbox/extras/models/models.py | 4 +- netbox/extras/querysets.py | 68 +++++++++++++++++------------ netbox/utilities/query_functions.py | 10 +++++ netbox/virtualization/api/views.py | 15 +++++++ netbox/virtualization/models.py | 3 +- netbox/virtualization/views.py | 2 +- 9 files changed, 87 insertions(+), 49 deletions(-) diff --git a/netbox/dcim/api/views.py b/netbox/dcim/api/views.py index 1967b5a3e..956b0cbff 100644 --- a/netbox/dcim/api/views.py +++ b/netbox/dcim/api/views.py @@ -340,20 +340,24 @@ class DeviceViewSet(CustomFieldModelViewSet): queryset = Device.objects.prefetch_related( 'device_type__manufacturer', 'device_role', 'tenant', 'platform', 'site', 'rack', 'parent_bay', 'virtual_chassis__master', 'primary_ip4__nat_outside', 'primary_ip6__nat_outside', 'tags', - ).add_config_context_annotation() - - #queryset = Device.objects.annotate( - # config_contexts=Subquery( - # ConfigContext.objects.filter( - # Q(sites=OuterRef('site')) | Q(sites=None) - # ).annotate( - # _data=EmptyGroupByJSONBAgg('data') - # ).values("_data") - # ) - #) - + ) filterset_class = filters.DeviceFilterSet + def get_queryset(self): + """ + Build the proper queryset based on the request context + + If the `brief` query param equates to True or the `exclude` query param + includes `config_context` as a value, return the base queryset. + + Else, return the queryset annotated with config context data + """ + + request = self.get_serializer_context()['request'] + if request.query_params.get('brief') or 'config_context' in request.query_params.get('exclude', []): + return self.queryset + return self.queryset.annotate_config_context_data() + def get_serializer_class(self): """ Select the specific serializer based on the request context. diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py index 413ba73c0..6fe2cea2f 100644 --- a/netbox/dcim/models/devices.py +++ b/netbox/dcim/models/devices.py @@ -15,7 +15,7 @@ from taggit.managers import TaggableManager from dcim.choices import * from dcim.constants import * from extras.models import ChangeLoggedModel, ConfigContextModel, CustomFieldModel, TaggedItem -from extras.querysets import ConfigContextQuerySetMixin +from extras.querysets import ConfigContextModelQuerySet from extras.utils import extras_features from utilities.choices import ColorChoices from utilities.fields import ColorField, NaturalOrderingField @@ -595,7 +595,7 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel): ) tags = TaggableManager(through=TaggedItem) - objects = ConfigContextQuerySetMixin.as_manager() + objects = ConfigContextModelQuerySet.as_manager() csv_headers = [ 'name', 'device_role', 'tenant', 'manufacturer', 'device_type', 'platform', 'serial', 'asset_tag', 'status', diff --git a/netbox/dcim/views.py b/netbox/dcim/views.py index c6c0ced97..886f3e702 100644 --- a/netbox/dcim/views.py +++ b/netbox/dcim/views.py @@ -1163,7 +1163,7 @@ class DeviceConfigView(ObjectView): class DeviceConfigContextView(ObjectConfigContextView): - queryset = Device.objects.all() + queryset = Device.objects.annotate_config_context_data() base_template = 'dcim/device.html' diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index e5577b5a9..efb165a06 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -542,10 +542,8 @@ class ConfigContextModel(models.Model): # Compile all config data, overwriting lower-weight values with higher-weight values where a collision occurs data = OrderedDict() - #for context in ConfigContext.objects.get_for_object(self): - # data = deepmerge(data, context.data) - for context in self.config_contexts: + for context in self.config_context_data: data = deepmerge(data, context) # If the object has local config context data defined, merge it last diff --git a/netbox/extras/querysets.py b/netbox/extras/querysets.py index c8b15f5a2..dea03bccd 100644 --- a/netbox/extras/querysets.py +++ b/netbox/extras/querysets.py @@ -1,8 +1,8 @@ from collections import OrderedDict -from django.contrib.postgres.aggregates import JSONBAgg from django.db.models import OuterRef, Subquery, Q, QuerySet +from utilities.query_functions import EmptyGroupByJSONBAgg from utilities.querysets import RestrictedQuerySet @@ -60,18 +60,14 @@ class ConfigContextQuerySet(RestrictedQuerySet): ).order_by('weight', 'name') -class EmptyGroupByJSONBAgg(JSONBAgg): - contains_aggregate = False +class ConfigContextModelQuerySet(RestrictedQuerySet): - -class ConfigContextQuerySetMixin(RestrictedQuerySet): - - def add_config_context_annotation(self): + def annotate_config_context_data(self): from extras.models import ConfigContext return self.annotate( - config_contexts=Subquery( + config_context_data=Subquery( ConfigContext.objects.filter( - self._add_config_context_filters() + self._get_config_context_filters() ).order_by( 'weight', 'name' @@ -81,28 +77,42 @@ class ConfigContextQuerySetMixin(RestrictedQuerySet): ) ) - def _add_config_context_filters(self): + def _get_config_context_filters(self): + base_query = Q( + Q(platforms=OuterRef('platform')) | Q(platforms=None), + Q(tenant_groups=OuterRef('tenant__group')) | Q(tenant_groups=None), + Q(tenants=OuterRef('tenant')) | Q(tenants=None), + Q(tags=OuterRef('tags')) | Q(tags=None), + is_active=True, + ) if self.model._meta.model_name == 'device': - return Q( - Q(sites=OuterRef('site')) | Q(sites=None), - Q(roles=OuterRef('device_role')) | Q(roles=None), - Q(platforms=OuterRef('platform')) | Q(platforms=None), - Q(tenant_groups=OuterRef('tenant__group')) | Q(tenant_groups=None), - Q(tenants=OuterRef('tenant')) | Q(tenants=None), - Q(tags=OuterRef('tags')) | Q(tags=None), - is_active=True, + base_query.add((Q(roles=OuterRef('device_role')) | Q(roles=None)), Q.AND) + base_query.add( + (Q( + regions__tree_id=OuterRef('site__region__tree_id'), + regions__level__lte=OuterRef('site__region__level'), + regions__lft__lte=OuterRef('site__region__lft'), + regions__rght__gte=OuterRef('site__region__rght'), + ) | Q(regions=None)), + Q.AND ) - else: - return Q( - Q(sites=OuterRef('site')) | Q(sites=None), - Q(roles=OuterRef('role')) | Q(roles=None), - Q(platforms=OuterRef('platform')) | Q(platforms=None), - Q(cluster_groups=OuterRef('cluster__group')) | Q(cluster_groups=None), - Q(clusters=OuterRef('cluster')) | Q(clusters=None), - Q(tenant_groups=OuterRef('tenant__group')) | Q(tenant_groups=None), - Q(tenants=OuterRef('tenant')) | Q(tenants=None), - Q(tags=OuterRef('tags')) | Q(tags=None), - is_active=True, + base_query.add((Q(sites=OuterRef('site')) | Q(sites=None)), Q.AND) + + elif self.model._meta.model_name == 'virtualmachine': + base_query.add((Q(roles=OuterRef('role')) | Q(roles=None)), Q.AND) + base_query.add((Q(cluster_groups=OuterRef('cluster__group')) | Q(cluster_groups=None)), Q.AND) + base_query.add((Q(clusters=OuterRef('cluster')) | Q(clusters=None)), Q.AND) + base_query.add( + (Q( + regions__tree_id=OuterRef('cluster__site__region__tree_id'), + regions__level__lte=OuterRef('cluster__site__region__level'), + regions__lft__lte=OuterRef('cluster__site__region__lft'), + regions__rght__gte=OuterRef('cluster__site__region__rght'), + ) | Q(regions=None)), + Q.AND ) + base_query.add((Q(sites=OuterRef('cluster__site')) | Q(sites=None)), Q.AND) + + return base_query diff --git a/netbox/utilities/query_functions.py b/netbox/utilities/query_functions.py index ee4310ea7..8ad7ceead 100644 --- a/netbox/utilities/query_functions.py +++ b/netbox/utilities/query_functions.py @@ -1,3 +1,4 @@ +from django.contrib.postgres.aggregates import JSONBAgg from django.db.models import F, Func @@ -7,3 +8,12 @@ class CollateAsChar(Func): """ function = 'C' template = '(%(expressions)s) COLLATE "%(function)s"' + + +class EmptyGroupByJSONBAgg(JSONBAgg): + """ + JSONBAgg is a builtin aggregation function which means it includes the use of a GROUP BY clause. + When used as an annotation for collecting config context data objects, the GROUP BY is + incorrect. This subclass overrides the Django ORM aggregation control to remove the GROUP BY. + """ + contains_aggregate = False diff --git a/netbox/virtualization/api/views.py b/netbox/virtualization/api/views.py index 1bf41c2b7..d19f0f9fa 100644 --- a/netbox/virtualization/api/views.py +++ b/netbox/virtualization/api/views.py @@ -64,6 +64,21 @@ class VirtualMachineViewSet(CustomFieldModelViewSet): ) filterset_class = filters.VirtualMachineFilterSet + def get_queryset(self): + """ + Build the proper queryset based on the request context + + If the `brief` query param equates to True or the `exclude` query param + includes `config_context` as a value, return the base queryset. + + Else, return the queryset annotated with config context data + """ + + request = self.get_serializer_context()['request'] + if request.query_params.get('brief') or 'config_context' in request.query_params.get('exclude', []): + return self.queryset + return self.queryset.annotate_config_context_data() + def get_serializer_class(self): """ Select the specific serializer based on the request context. diff --git a/netbox/virtualization/models.py b/netbox/virtualization/models.py index 7d0b99872..39f579e30 100644 --- a/netbox/virtualization/models.py +++ b/netbox/virtualization/models.py @@ -8,6 +8,7 @@ from taggit.managers import TaggableManager from dcim.choices import InterfaceModeChoices from dcim.models import BaseInterface, Device from extras.models import ChangeLoggedModel, ConfigContextModel, CustomFieldModel, ObjectChange, TaggedItem +from extras.querysets import ConfigContextModelQuerySet from extras.utils import extras_features from utilities.fields import NaturalOrderingField from utilities.ordering import naturalize_interface @@ -282,7 +283,7 @@ class VirtualMachine(ChangeLoggedModel, ConfigContextModel, CustomFieldModel): ) tags = TaggableManager(through=TaggedItem) - objects = RestrictedQuerySet.as_manager() + objects = ConfigContextModelQuerySet.as_manager() csv_headers = [ 'name', 'status', 'role', 'cluster', 'tenant', 'platform', 'vcpus', 'memory', 'disk', 'comments', diff --git a/netbox/virtualization/views.py b/netbox/virtualization/views.py index a06a2e5ff..4cabfedeb 100644 --- a/netbox/virtualization/views.py +++ b/netbox/virtualization/views.py @@ -261,7 +261,7 @@ class VirtualMachineView(ObjectView): class VirtualMachineConfigContextView(ObjectConfigContextView): - queryset = VirtualMachine.objects.all() + queryset = VirtualMachine.objects.annotate_config_context_data() base_template = 'virtualization/virtualmachine.html'