Merge branch 'develop' into develop-2.10

netbox/extras/api/views.py

@@ -23,6 +23,29 @@ from utilities.utils import copy_safe_request
 from . import serializers
 
 
+class ConfigContextQuerySetMixin:
+    """
+    Used by views that work with config context models (device and virtual machine).
+    Provides a get_queryset() method which deals with adding the config context
+    data annotation or not.
+    """
+
+    def get_queryset(self):
+        """
+        Build the proper queryset based on the request context
+
+        If the `brief` query param equates to True or the `exclude` query param
+        includes `config_context` as a value, return the base queryset.
+
+        Else, return the queryset annotated with config context data
+        """
+
+        request = self.get_serializer_context()['request']
+        if request.query_params.get('brief') or 'config_context' in request.query_params.get('exclude', []):
+            return self.queryset
+        return self.queryset.annotate_config_context_data()
+
+
 class ExtrasRootView(APIRootView):
     """
     Extras API root view

netbox/extras/models/models.py

@@ -465,8 +465,16 @@ class ConfigContextModel(models.Model):
 
         # Compile all config data, overwriting lower-weight values with higher-weight values where a collision occurs
         data = OrderedDict()
-        for context in ConfigContext.objects.get_for_object(self):
-            data = deepmerge(data, context.data)
+
+        if not hasattr(self, 'config_context_data'):
+            # The annotation is not available, so we fall back to manually querying for the config context objects
+            config_context_data = ConfigContext.objects.get_for_object(self, aggregate_data=True)
+        else:
+            # The attribute may exist, but the annotated value could be None if there is no config context data
+            config_context_data = self.config_context_data or []
+
+        for context in config_context_data:
+            data = deepmerge(data, context)
 
         # If the object has local config context data defined, merge it last
         if self.local_context_data:

netbox/extras/plugins/views.py

@@ -10,8 +10,6 @@ from rest_framework.response import Response
 from rest_framework.reverse import reverse
 from rest_framework.views import APIView
 
-from extras.plugins.utils import import_object
-
 
 class InstalledPluginsAdminView(View):
     """
@@ -62,11 +60,7 @@ class PluginsAPIRootView(APIView):
     @staticmethod
     def _get_plugin_entry(plugin, app_config, request, format):
         # Check if the plugin specifies any API URLs
-        api_app_name = import_object(f"{plugin}.api.urls.app_name")
-        if api_app_name is None:
-            # Plugin does not expose an API
-            return None
-
+        api_app_name = f'{app_config.name}-api'
         try:
             entry = (getattr(app_config, 'base_url', app_config.label), reverse(
                 f"plugins-api:{api_app_name}:api-root",

netbox/extras/querysets.py

@@ -1,13 +1,17 @@
-from django.db.models import Q
+from django.db.models import OuterRef, Subquery, Q
 
+from utilities.query_functions import EmptyGroupByJSONBAgg, OrderableJSONBAgg
 from utilities.querysets import RestrictedQuerySet
 
 
 class ConfigContextQuerySet(RestrictedQuerySet):
 
-    def get_for_object(self, obj):
+    def get_for_object(self, obj, aggregate_data=False):
         """
         Return all applicable ConfigContexts for a given object. Only active ConfigContexts will be included.
+
+        Args:
+          aggregate_data: If True, use the JSONBAgg aggregate function to return only the list of JSON data objects
         """
 
         # `device_role` for Device; `role` for VirtualMachine
@@ -27,7 +31,7 @@ class ConfigContextQuerySet(RestrictedQuerySet):
         else:
             regions = []
 
-        return self.filter(
+        queryset = self.filter(
             Q(regions__in=regions) | Q(regions=None),
             Q(sites=obj.site) | Q(sites=None),
             Q(roles=role) | Q(roles=None),
@@ -39,3 +43,72 @@ class ConfigContextQuerySet(RestrictedQuerySet):
             Q(tags__slug__in=obj.tags.slugs()) | Q(tags=None),
             is_active=True,
         ).order_by('weight', 'name')
+
+        if aggregate_data:
+            return queryset.aggregate(
+                config_context_data=OrderableJSONBAgg('data', ordering=['weight', 'name'])
+            )['config_context_data']
+
+        return queryset
+
+
+class ConfigContextModelQuerySet(RestrictedQuerySet):
+    """
+    QuerySet manager used by models which support ConfigContext (device and virtual machine).
+
+    Includes a method which appends an annotation of aggregated config context JSON data objects. This is
+    implemented as a subquery which performs all the joins necessary to filter relevant config context objects.
+    This offers a substantial performance gain over ConfigContextQuerySet.get_for_object() when dealing with
+    multiple objects.
+
+    This allows the annotation to be entirely optional.
+    """
+
+    def annotate_config_context_data(self):
+        """
+        Attach the subquery annotation to the base queryset
+        """
+        from extras.models import ConfigContext
+        return self.annotate(
+            config_context_data=Subquery(
+                ConfigContext.objects.filter(
+                    self._get_config_context_filters()
+                ).annotate(
+                    _data=EmptyGroupByJSONBAgg('data', ordering=['weight', 'name'])
+                ).values("_data")
+            )
+        )
+
+    def _get_config_context_filters(self):
+        # Construct the set of Q objects for the specific object types
+        base_query = Q(
+            Q(platforms=OuterRef('platform')) | Q(platforms=None),
+            Q(tenant_groups=OuterRef('tenant__group')) | Q(tenant_groups=None),
+            Q(tenants=OuterRef('tenant')) | Q(tenants=None),
+            Q(tags=OuterRef('tags')) | Q(tags=None),
+            is_active=True,
+        )
+
+        if self.model._meta.model_name == 'device':
+            base_query.add((Q(roles=OuterRef('device_role')) | Q(roles=None)), Q.AND)
+            base_query.add((Q(sites=OuterRef('site')) | Q(sites=None)), Q.AND)
+            region_field = 'site__region'
+
+        elif self.model._meta.model_name == 'virtualmachine':
+            base_query.add((Q(roles=OuterRef('role')) | Q(roles=None)), Q.AND)
+            base_query.add((Q(cluster_groups=OuterRef('cluster__group')) | Q(cluster_groups=None)), Q.AND)
+            base_query.add((Q(clusters=OuterRef('cluster')) | Q(clusters=None)), Q.AND)
+            base_query.add((Q(sites=OuterRef('cluster__site')) | Q(sites=None)), Q.AND)
+            region_field = 'cluster__site__region'
+
+        base_query.add(
+            (Q(
+                regions__tree_id=OuterRef(f'{region_field}__tree_id'),
+                regions__level__lte=OuterRef(f'{region_field}__level'),
+                regions__lft__lte=OuterRef(f'{region_field}__lft'),
+                regions__rght__gte=OuterRef(f'{region_field}__rght'),
+            ) | Q(regions=None)),
+            Q.AND
+        )
+
+        return base_query

netbox/extras/tests/test_models.py

@@ -1,6 +1,9 @@
 from django.test import TestCase
 
-from extras.models import Tag
+from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Platform, Site, Region
+from extras.models import ConfigContext, Tag
+from tenancy.models import Tenant, TenantGroup
+from virtualization.models import Cluster, ClusterGroup, ClusterType, VirtualMachine
 
 
 class TagTest(TestCase):
@@ -10,3 +13,276 @@ class TagTest(TestCase):
         tag.save()
 
         self.assertEqual(tag.slug, 'testing-unicode-台灣')
+
+
+class ConfigContextTest(TestCase):
+    """
+    These test cases deal with the weighting, ordering, and deep merge logic of config context data.
+
+    It also ensures the various config context querysets are consistent.
+    """
+
+    def setUp(self):
+
+        manufacturer = Manufacturer.objects.create(name='Manufacturer 1', slug='manufacturer-1')
+        self.devicetype = DeviceType.objects.create(manufacturer=manufacturer, model='Device Type 1', slug='device-type-1')
+        self.devicerole = DeviceRole.objects.create(name='Device Role 1', slug='device-role-1')
+        self.region = Region.objects.create(name="Region")
+        self.site = Site.objects.create(name='Site-1', slug='site-1', region=self.region)
+        self.platform = Platform.objects.create(name="Platform")
+        self.tenantgroup = TenantGroup.objects.create(name="Tenant Group")
+        self.tenant = Tenant.objects.create(name="Tenant", group=self.tenantgroup)
+        self.tag = Tag.objects.create(name="Tag", slug="tag")
+
+        self.device = Device.objects.create(
+            name='Device 1',
+            device_type=self.devicetype,
+            device_role=self.devicerole,
+            site=self.site
+        )
+
+    def test_higher_weight_wins(self):
+
+        context1 = ConfigContext(
+            name="context 1",
+            weight=101,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 777
+            }
+        )
+        context2 = ConfigContext(
+            name="context 2",
+            weight=100,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 789
+            }
+        )
+        ConfigContext.objects.bulk_create([context1, context2])
+
+        expected_data = {
+            "a": 123,
+            "b": 456,
+            "c": 777
+        }
+        self.assertEqual(self.device.get_config_context(), expected_data)
+
+    def test_name_ordering_after_weight(self):
+
+        context1 = ConfigContext(
+            name="context 1",
+            weight=100,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 777
+            }
+        )
+        context2 = ConfigContext(
+            name="context 2",
+            weight=100,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 789
+            }
+        )
+        ConfigContext.objects.bulk_create([context1, context2])
+
+        expected_data = {
+            "a": 123,
+            "b": 456,
+            "c": 789
+        }
+        self.assertEqual(self.device.get_config_context(), expected_data)
+
+    def test_annotation_same_as_get_for_object(self):
+        """
+        This test incorperates features from all of the above tests cases to ensure
+        the annotate_config_context_data() and get_for_object() queryset methods are the same.
+        """
+        context1 = ConfigContext(
+            name="context 1",
+            weight=101,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 777
+            }
+        )
+        context2 = ConfigContext(
+            name="context 2",
+            weight=100,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 789
+            }
+        )
+        context3 = ConfigContext(
+            name="context 3",
+            weight=99,
+            data={
+                "d": 1
+            }
+        )
+        context4 = ConfigContext(
+            name="context 4",
+            weight=99,
+            data={
+                "d": 2
+            }
+        )
+        ConfigContext.objects.bulk_create([context1, context2, context3, context4])
+
+        annotated_queryset = Device.objects.filter(name=self.device.name).annotate_config_context_data()
+        self.assertEqual(self.device.get_config_context(), annotated_queryset[0].get_config_context())
+
+    def test_annotation_same_as_get_for_object_device_relations(self):
+
+        site_context = ConfigContext.objects.create(
+            name="site",
+            weight=100,
+            data={
+                "site": 1
+            }
+        )
+        site_context.sites.add(self.site)
+        region_context = ConfigContext.objects.create(
+            name="region",
+            weight=100,
+            data={
+                "region": 1
+            }
+        )
+        region_context.regions.add(self.region)
+        platform_context = ConfigContext.objects.create(
+            name="platform",
+            weight=100,
+            data={
+                "platform": 1
+            }
+        )
+        platform_context.platforms.add(self.platform)
+        tenant_group_context = ConfigContext.objects.create(
+            name="tenant group",
+            weight=100,
+            data={
+                "tenant_group": 1
+            }
+        )
+        tenant_group_context.tenant_groups.add(self.tenantgroup)
+        tenant_context = ConfigContext.objects.create(
+            name="tenant",
+            weight=100,
+            data={
+                "tenant": 1
+            }
+        )
+        tenant_context.tenants.add(self.tenant)
+        tag_context = ConfigContext.objects.create(
+            name="tag",
+            weight=100,
+            data={
+                "tag": 1
+            }
+        )
+        tag_context.tags.add(self.tag)
+
+        device = Device.objects.create(
+            name="Device 2",
+            site=self.site,
+            tenant=self.tenant,
+            platform=self.platform,
+            device_role=self.devicerole,
+            device_type=self.devicetype
+        )
+        device.tags.add(self.tag)
+
+        annotated_queryset = Device.objects.filter(name=device.name).annotate_config_context_data()
+        self.assertEqual(device.get_config_context(), annotated_queryset[0].get_config_context())
+
+    def test_annotation_same_as_get_for_object_virtualmachine_relations(self):
+
+        site_context = ConfigContext.objects.create(
+            name="site",
+            weight=100,
+            data={
+                "site": 1
+            }
+        )
+        site_context.sites.add(self.site)
+        region_context = ConfigContext.objects.create(
+            name="region",
+            weight=100,
+            data={
+                "region": 1
+            }
+        )
+        region_context.regions.add(self.region)
+        platform_context = ConfigContext.objects.create(
+            name="platform",
+            weight=100,
+            data={
+                "platform": 1
+            }
+        )
+        platform_context.platforms.add(self.platform)
+        tenant_group_context = ConfigContext.objects.create(
+            name="tenant group",
+            weight=100,
+            data={
+                "tenant_group": 1
+            }
+        )
+        tenant_group_context.tenant_groups.add(self.tenantgroup)
+        tenant_context = ConfigContext.objects.create(
+            name="tenant",
+            weight=100,
+            data={
+                "tenant": 1
+            }
+        )
+        tenant_context.tenants.add(self.tenant)
+        tag_context = ConfigContext.objects.create(
+            name="tag",
+            weight=100,
+            data={
+                "tag": 1
+            }
+        )
+        tag_context.tags.add(self.tag)
+        cluster_group = ClusterGroup.objects.create(name="Cluster Group")
+        cluster_group_context = ConfigContext.objects.create(
+            name="cluster group",
+            weight=100,
+            data={
+                "cluster_group": 1
+            }
+        )
+        cluster_group_context.cluster_groups.add(cluster_group)
+        cluster_type = ClusterType.objects.create(name="Cluster Type 1")
+        cluster = Cluster.objects.create(name="Cluster", group=cluster_group, type=cluster_type)
+        cluster_context = ConfigContext.objects.create(
+            name="cluster",
+            weight=100,
+            data={
+                "cluster": 1
+            }
+        )
+        cluster_context.clusters.add(cluster)
+
+        virtual_machine = VirtualMachine.objects.create(
+            name="VM 1",
+            cluster=cluster,
+            tenant=self.tenant,
+            platform=self.platform,
+            role=self.devicerole
+        )
+        virtual_machine.tags.add(self.tag)
+
+        annotated_queryset = VirtualMachine.objects.filter(name=virtual_machine.name).annotate_config_context_data()
+        self.assertEqual(virtual_machine.get_config_context(), annotated_queryset[0].get_config_context())