From 3ba18633de14522e48dd9ac50c27ad308a18bb7c Mon Sep 17 00:00:00 2001
From: John Anderson <lampwins@gmail.com>
Date: Tue, 20 Oct 2020 01:07:22 -0400
Subject: [PATCH] initial work on config context performance improvements

---
 netbox/dcim/api/views.py       | 13 ++++++++-
 netbox/dcim/models/devices.py  |  3 +-
 netbox/extras/models/models.py |  7 +++--
 netbox/extras/querysets.py     | 51 +++++++++++++++++++++++++++++++++-
 4 files changed, 69 insertions(+), 5 deletions(-)

diff --git a/netbox/dcim/api/views.py b/netbox/dcim/api/views.py
index f5b37021d..1967b5a3e 100644
--- a/netbox/dcim/api/views.py
+++ b/netbox/dcim/api/views.py
@@ -340,7 +340,18 @@ class DeviceViewSet(CustomFieldModelViewSet):
     queryset = Device.objects.prefetch_related(
         'device_type__manufacturer', 'device_role', 'tenant', 'platform', 'site', 'rack', 'parent_bay',
         'virtual_chassis__master', 'primary_ip4__nat_outside', 'primary_ip6__nat_outside', 'tags',
-    )
+    ).add_config_context_annotation()
+
+    #queryset = Device.objects.annotate(
+    #    config_contexts=Subquery(
+    #        ConfigContext.objects.filter(
+    #            Q(sites=OuterRef('site')) | Q(sites=None)
+    #        ).annotate(
+    #            _data=EmptyGroupByJSONBAgg('data')
+    #        ).values("_data")
+    #    )
+    #)
+
     filterset_class = filters.DeviceFilterSet
 
     def get_serializer_class(self):
diff --git a/netbox/dcim/models/devices.py b/netbox/dcim/models/devices.py
index e96becadf..413ba73c0 100644
--- a/netbox/dcim/models/devices.py
+++ b/netbox/dcim/models/devices.py
@@ -15,6 +15,7 @@ from taggit.managers import TaggableManager
 from dcim.choices import *
 from dcim.constants import *
 from extras.models import ChangeLoggedModel, ConfigContextModel, CustomFieldModel, TaggedItem
+from extras.querysets import ConfigContextQuerySetMixin
 from extras.utils import extras_features
 from utilities.choices import ColorChoices
 from utilities.fields import ColorField, NaturalOrderingField
@@ -594,7 +595,7 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel):
     )
     tags = TaggableManager(through=TaggedItem)
 
-    objects = RestrictedQuerySet.as_manager()
+    objects = ConfigContextQuerySetMixin.as_manager()
 
     csv_headers = [
         'name', 'device_role', 'tenant', 'manufacturer', 'device_type', 'platform', 'serial', 'asset_tag', 'status',
diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py
index e57caf091..e5577b5a9 100644
--- a/netbox/extras/models/models.py
+++ b/netbox/extras/models/models.py
@@ -542,8 +542,11 @@ class ConfigContextModel(models.Model):
 
         # Compile all config data, overwriting lower-weight values with higher-weight values where a collision occurs
         data = OrderedDict()
-        for context in ConfigContext.objects.get_for_object(self):
-            data = deepmerge(data, context.data)
+        #for context in ConfigContext.objects.get_for_object(self):
+        #    data = deepmerge(data, context.data)
+
+        for context in self.config_contexts:
+            data = deepmerge(data, context)
 
         # If the object has local config context data defined, merge it last
         if self.local_context_data:
diff --git a/netbox/extras/querysets.py b/netbox/extras/querysets.py
index 9d9b55778..c8b15f5a2 100644
--- a/netbox/extras/querysets.py
+++ b/netbox/extras/querysets.py
@@ -1,6 +1,7 @@
 from collections import OrderedDict
 
-from django.db.models import Q, QuerySet
+from django.contrib.postgres.aggregates import JSONBAgg
+from django.db.models import OuterRef, Subquery, Q, QuerySet
 
 from utilities.querysets import RestrictedQuerySet
 
@@ -57,3 +58,51 @@ class ConfigContextQuerySet(RestrictedQuerySet):
             Q(tags__slug__in=obj.tags.slugs()) | Q(tags=None),
             is_active=True,
         ).order_by('weight', 'name')
+
+
+class EmptyGroupByJSONBAgg(JSONBAgg):
+    contains_aggregate = False
+
+
+class ConfigContextQuerySetMixin(RestrictedQuerySet):
+
+    def add_config_context_annotation(self):
+        from extras.models import ConfigContext
+        return self.annotate(
+            config_contexts=Subquery(
+                ConfigContext.objects.filter(
+                    self._add_config_context_filters()
+                ).order_by(
+                    'weight',
+                    'name'
+                ).annotate(
+                    _data=EmptyGroupByJSONBAgg('data')
+                ).values("_data")
+            )
+        )
+
+    def _add_config_context_filters(self):
+
+
+        if self.model._meta.model_name == 'device':
+            return Q(
+                Q(sites=OuterRef('site')) | Q(sites=None),
+                Q(roles=OuterRef('device_role')) | Q(roles=None),
+                Q(platforms=OuterRef('platform')) | Q(platforms=None),
+                Q(tenant_groups=OuterRef('tenant__group')) | Q(tenant_groups=None),
+                Q(tenants=OuterRef('tenant')) | Q(tenants=None),
+                Q(tags=OuterRef('tags')) | Q(tags=None),
+                is_active=True,
+            )
+        else:
+            return Q(
+                Q(sites=OuterRef('site')) | Q(sites=None),
+                Q(roles=OuterRef('role')) | Q(roles=None),
+                Q(platforms=OuterRef('platform')) | Q(platforms=None),
+                Q(cluster_groups=OuterRef('cluster__group')) | Q(cluster_groups=None),
+                Q(clusters=OuterRef('cluster')) | Q(clusters=None),
+                Q(tenant_groups=OuterRef('tenant__group')) | Q(tenant_groups=None),
+                Q(tenants=OuterRef('tenant')) | Q(tenants=None),
+                Q(tags=OuterRef('tags')) | Q(tags=None),
+                is_active=True,
+            )