Merge pull request #3080 from digitalocean/2647-cacheops

change cacheing to use cacheops

base_requirements.txt

@@ -2,6 +2,10 @@
 # https://github.com/django/django
 Django
 
+# Django caching using Redis
+# https://github.com/Suor/django-cacheops
+django-cacheops
+
 # Django middleware which permits cross-domain API requests
 # https://github.com/OttoYiu/django-cors-headers
 django-cors-headers
@@ -18,10 +22,6 @@ django-filter
 # https://github.com/django-mptt/django-mptt
 django-mptt
 
-# Django caching using Redis
-# https://github.com/niwinz/django-redis
-django-redis
-
 # Abstraction models for rendering and paginating HTML tables
 # https://github.com/jieter/django-tables2
 django-tables2

netbox/dcim/views.py

@@ -197,7 +197,6 @@ class SiteListView(PermissionRequiredMixin, ObjectListView):
 class SiteView(PermissionRequiredMixin, View):
     permission_required = 'dcim.view_site'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, slug):
 
         site = get_object_or_404(Site.objects.select_related('region', 'tenant__group'), slug=slug)
@@ -356,7 +355,6 @@ class RackElevationListView(PermissionRequiredMixin, View):
     """
     permission_required = 'dcim.view_rack'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request):
 
         racks = Rack.objects.select_related(
@@ -396,7 +394,6 @@ class RackElevationListView(PermissionRequiredMixin, View):
 class RackView(PermissionRequiredMixin, View):
     permission_required = 'dcim.view_rack'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         rack = get_object_or_404(Rack.objects.select_related('site__region', 'tenant__group', 'group', 'role'), pk=pk)
@@ -575,7 +572,6 @@ class DeviceTypeListView(PermissionRequiredMixin, ObjectListView):
 class DeviceTypeView(PermissionRequiredMixin, View):
     permission_required = 'dcim.view_devicetype'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         devicetype = get_object_or_404(DeviceType, pk=pk)
@@ -916,7 +912,6 @@ class DeviceListView(PermissionRequiredMixin, ObjectListView):
 class DeviceView(PermissionRequiredMixin, View):
     permission_required = 'dcim.view_device'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         device = get_object_or_404(Device.objects.select_related(
@@ -998,7 +993,6 @@ class DeviceView(PermissionRequiredMixin, View):
 class DeviceInventoryView(PermissionRequiredMixin, View):
     permission_required = 'dcim.view_device'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         device = get_object_or_404(Device, pk=pk)
@@ -1020,7 +1014,6 @@ class DeviceInventoryView(PermissionRequiredMixin, View):
 class DeviceStatusView(PermissionRequiredMixin, View):
     permission_required = ('dcim.view_device', 'dcim.napalm_read')
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         device = get_object_or_404(Device, pk=pk)
@@ -1034,7 +1027,6 @@ class DeviceStatusView(PermissionRequiredMixin, View):
 class DeviceLLDPNeighborsView(PermissionRequiredMixin, View):
     permission_required = ('dcim.view_device', 'dcim.napalm_read')
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         device = get_object_or_404(Device, pk=pk)
@@ -1052,7 +1044,6 @@ class DeviceLLDPNeighborsView(PermissionRequiredMixin, View):
 class DeviceConfigView(PermissionRequiredMixin, View):
     permission_required = ('dcim.view_device', 'dcim.napalm_read')
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         device = get_object_or_404(Device, pk=pk)
@@ -1290,7 +1281,6 @@ class PowerOutletBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
 class InterfaceView(PermissionRequiredMixin, View):
     permission_required = 'dcim.view_interface'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         interface = get_object_or_404(Interface, pk=pk)
@@ -1511,7 +1501,6 @@ class DeviceBayDeleteView(PermissionRequiredMixin, ObjectDeleteView):
 class DeviceBayPopulateView(PermissionRequiredMixin, View):
     permission_required = 'dcim.change_devicebay'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         device_bay = get_object_or_404(DeviceBay, pk=pk)
@@ -1546,7 +1535,6 @@ class DeviceBayPopulateView(PermissionRequiredMixin, View):
 class DeviceBayDepopulateView(PermissionRequiredMixin, View):
     permission_required = 'dcim.change_devicebay'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         device_bay = get_object_or_404(DeviceBay, pk=pk)
@@ -1686,7 +1674,6 @@ class CableListView(PermissionRequiredMixin, ObjectListView):
 class CableView(PermissionRequiredMixin, View):
     permission_required = 'dcim.view_cable'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         cable = get_object_or_404(Cable, pk=pk)
@@ -1702,7 +1689,6 @@ class CableTraceView(PermissionRequiredMixin, View):
     """
     permission_required = 'dcim.view_cable'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, model, pk):
 
         obj = get_object_or_404(model, pk=pk)
@@ -1742,7 +1728,6 @@ class CableCreateView(PermissionRequiredMixin, GetReturnURLMixin, View):
 
         return super().dispatch(request, *args, **kwargs)
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, *args, **kwargs):
 
         # Parse initial data manually to avoid setting field values as lists
@@ -2059,7 +2044,6 @@ class VirtualChassisCreateView(PermissionRequiredMixin, View):
 class VirtualChassisEditView(PermissionRequiredMixin, GetReturnURLMixin, View):
     permission_required = 'dcim.change_virtualchassis'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         virtual_chassis = get_object_or_404(VirtualChassis, pk=pk)
@@ -2128,7 +2112,6 @@ class VirtualChassisDeleteView(PermissionRequiredMixin, ObjectDeleteView):
 class VirtualChassisAddMemberView(PermissionRequiredMixin, GetReturnURLMixin, View):
     permission_required = 'dcim.change_virtualchassis'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         virtual_chassis = get_object_or_404(VirtualChassis, pk=pk)
@@ -2183,7 +2166,6 @@ class VirtualChassisAddMemberView(PermissionRequiredMixin, GetReturnURLMixin, Vi
 class VirtualChassisRemoveMemberView(PermissionRequiredMixin, GetReturnURLMixin, View):
     permission_required = 'dcim.change_virtualchassis'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         device = get_object_or_404(Device, pk=pk, virtual_chassis__isnull=False)
@@ -2247,7 +2229,6 @@ class PowerPanelListView(PermissionRequiredMixin, ObjectListView):
 class PowerPanelView(PermissionRequiredMixin, View):
     permission_required = 'dcim.view_powerpanel'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         powerpanel = get_object_or_404(PowerPanel.objects.select_related('site', 'rack_group'), pk=pk)
@@ -2317,7 +2298,6 @@ class PowerFeedListView(PermissionRequiredMixin, ObjectListView):
 class PowerFeedView(PermissionRequiredMixin, View):
     permission_required = 'dcim.view_powerfeed'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         powerfeed = get_object_or_404(PowerFeed.objects.select_related('power_panel', 'rack'), pk=pk)

netbox/extras/views.py

@@ -43,7 +43,6 @@ class TagListView(ObjectListView):
 
 class TagView(View):
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, slug):
 
         tag = get_object_or_404(Tag, slug=slug)
@@ -111,7 +110,6 @@ class ConfigContextListView(PermissionRequiredMixin, ObjectListView):
 class ConfigContextView(PermissionRequiredMixin, View):
     permission_required = 'extras.view_configcontext'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         configcontext = get_object_or_404(ConfigContext, pk=pk)
@@ -159,7 +157,6 @@ class ObjectConfigContextView(View):
     object_class = None
     base_template = None
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         obj = get_object_or_404(self.object_class, pk=pk)
@@ -192,7 +189,6 @@ class ObjectChangeListView(PermissionRequiredMixin, ObjectListView):
 class ObjectChangeView(PermissionRequiredMixin, View):
     permission_required = 'extras.view_objectchange'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         objectchange = get_object_or_404(ObjectChange, pk=pk)
@@ -215,7 +211,6 @@ class ObjectChangeLogView(View):
     Present a history of changes made to a particular object.
     """
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, model, **kwargs):
 
         # Get object my model and kwargs (e.g. slug='foo')
@@ -289,7 +284,6 @@ class ReportListView(PermissionRequiredMixin, View):
     """
     permission_required = 'extras.view_reportresult'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request):
 
         reports = get_reports()
@@ -314,7 +308,6 @@ class ReportView(PermissionRequiredMixin, View):
     """
     permission_required = 'extras.view_reportresult'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, name):
 
         # Retrieve the Report by "<module>.<report>"

netbox/ipam/views.py

@@ -127,7 +127,6 @@ class VRFListView(PermissionRequiredMixin, ObjectListView):
 class VRFView(PermissionRequiredMixin, View):
     permission_required = 'ipam.view_vrf'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         vrf = get_object_or_404(VRF.objects.all(), pk=pk)
@@ -322,7 +321,6 @@ class AggregateListView(PermissionRequiredMixin, ObjectListView):
 class AggregateView(PermissionRequiredMixin, View):
     permission_required = 'ipam.view_aggregate'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         aggregate = get_object_or_404(Aggregate, pk=pk)
@@ -460,7 +458,6 @@ class PrefixListView(PermissionRequiredMixin, ObjectListView):
 class PrefixView(PermissionRequiredMixin, View):
     permission_required = 'ipam.view_prefix'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         prefix = get_object_or_404(Prefix.objects.select_related(
@@ -505,7 +502,6 @@ class PrefixView(PermissionRequiredMixin, View):
 class PrefixPrefixesView(PermissionRequiredMixin, View):
     permission_required = 'ipam.view_prefix'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         prefix = get_object_or_404(Prefix.objects.all(), pk=pk)
@@ -549,7 +545,6 @@ class PrefixPrefixesView(PermissionRequiredMixin, View):
 class PrefixIPAddressesView(PermissionRequiredMixin, View):
     permission_required = 'ipam.view_prefix'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         prefix = get_object_or_404(Prefix.objects.all(), pk=pk)
@@ -650,7 +645,6 @@ class IPAddressListView(PermissionRequiredMixin, ObjectListView):
 class IPAddressView(PermissionRequiredMixin, View):
     permission_required = 'ipam.view_ipaddress'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         ipaddress = get_object_or_404(IPAddress.objects.select_related('vrf__tenant', 'tenant'), pk=pk)
@@ -734,7 +728,6 @@ class IPAddressAssignView(PermissionRequiredMixin, View):
 
         return super().dispatch(request, *args, **kwargs)
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request):
 
         form = forms.IPAddressAssignForm()
@@ -847,7 +840,6 @@ class VLANGroupBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
 class VLANGroupVLANsView(PermissionRequiredMixin, View):
     permission_required = 'ipam.view_vlangroup'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         vlan_group = get_object_or_404(VLANGroup.objects.all(), pk=pk)
@@ -898,7 +890,6 @@ class VLANListView(PermissionRequiredMixin, ObjectListView):
 class VLANView(PermissionRequiredMixin, View):
     permission_required = 'ipam.view_vlan'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         vlan = get_object_or_404(VLAN.objects.select_related(
@@ -917,7 +908,6 @@ class VLANView(PermissionRequiredMixin, View):
 class VLANMembersView(PermissionRequiredMixin, View):
     permission_required = 'ipam.view_vlan'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         vlan = get_object_or_404(VLAN.objects.all(), pk=pk)
@@ -996,7 +986,6 @@ class ServiceListView(PermissionRequiredMixin, ObjectListView):
 class ServiceView(PermissionRequiredMixin, View):
     permission_required = 'ipam.view_service'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         service = get_object_or_404(Service, pk=pk)

netbox/netbox/configuration.example.py

@@ -31,6 +31,7 @@ REDIS = {
     'PORT': 6379,
     'PASSWORD': '',
     'DATABASE': 0,
+    'CACHE_DATABASE': 1,
     'DEFAULT_TIMEOUT': 300,
     'SSL': False,
 }
@@ -60,16 +61,7 @@ BANNER_LOGIN = ''
 # BASE_PATH = 'netbox/'
 BASE_PATH = ''
 
-# The fraction of entries that are culled when CACHE_MAX_ENTRIES is reached. The actual ratio is 1 / CACHE_CULL_FREQUENCY,
-# so set CACHE_CULL_FREQUENCY to 2 to cull half the entries when CACHE_MAX_ENTRIES is reached. This setting should be an
-# integer and defaults to 3
-CACHE_CULL_FREQUENCY = 3
-
-# Max number of entries (unique pages) to store in the cache at a time.
-CACHE_MAX_ENTRIES = 300
-
-# Cache timeout in seconds. Set to `None` to enforce an infinate timeout. Set to 0 to dissable caching by immediatly
-# expiring keys. Defaults to 900 (15 minutes)
+# Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes)
 CACHE_TIMEOUT = 900
 
 # Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90)

netbox/netbox/settings.py

@@ -138,6 +138,7 @@ REDIS_HOST = REDIS.get('HOST', 'localhost')
 REDIS_PORT = REDIS.get('PORT', 6379)
 REDIS_PASSWORD = REDIS.get('PASSWORD', '')
 REDIS_DATABASE = REDIS.get('DATABASE', 0)
+REDIS_CACHE_DATABASE = REDIS.get('CACHE_DATABASE', 1)
 REDIS_DEFAULT_TIMEOUT = REDIS.get('DEFAULT_TIMEOUT', 300)
 REDIS_SSL = REDIS.get('SSL', False)
 
@@ -159,8 +160,8 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django.contrib.humanize',
+    'cacheops',
     'corsheaders',
-    'django_redis',
     'debug_toolbar',
     'django_filters',
     'django_tables2',
@@ -231,21 +232,30 @@ else:
 if REDIS_PASSWORD:
     REDIS_CACHE_CON_STRING = '{}@{}'.format(REDIS_PASSWORD, REDIS_CACHE_CON_STRING)
 
-REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format(REDIS_CACHE_CON_STRING, REDIS_HOST, REDIS_PORT, REDIS_DATABASE)
-CACHE_BACKEND = 'django_redis.cache.RedisCache'
+REDIS_CACHE_CON_STRING = '{}{}:{}/{}'.format(REDIS_CACHE_CON_STRING, REDIS_HOST, REDIS_PORT, REDIS_CACHE_DATABASE)
 
-CACHES = {
-    "default": {
-        "BACKEND": CACHE_BACKEND,
-        "LOCATION": REDIS_CACHE_CON_STRING,
-        'TIMEOUT': CACHE_TIMEOUT,
-        "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.DefaultClient",
-            "MAX_ENTRIES": CACHE_MAX_ENTRIES,
-            "CULL_FREQUENCY": CACHE_CULL_FREQUENCY
-        }
-    }
+if not CACHE_TIMEOUT:
+    CACHEOPS_ENABLED = False
+else:
+    CACHEOPS_ENABLED = True
+
+CACHEOPS_REDIS = REDIS_CACHE_CON_STRING
+CACHEOPS_DEFAULTS = {
+    'timeout': CACHE_TIMEOUT
 }
+CACHEOPS = {
+    'auth.user': {'ops': 'get', 'timeout': 60 * 15},
+    'auth.*': {'ops': ('fetch', 'get')},
+    'auth.permission': {'ops': 'all'},
+    'dcim.*': {'ops': 'all'},
+    'ipam.*': {'ops': 'all'},
+    'extras.*': {'ops': 'all'},
+    'secrets.*': {'ops': 'all'},
+    'users.*': {'ops': 'all'},
+    'tenancy.*': {'ops': 'all'},
+    'virtualization.*': {'ops': 'all'},
+}
+CACHEOPS_DEGRADE_ON_FAILURE = True
 
 # WSGI
 WSGI_APPLICATION = 'netbox.wsgi.application'

netbox/netbox/views.py

@@ -171,7 +171,6 @@ SEARCH_TYPES = OrderedDict((
 class HomeView(View):
     template_name = 'home.html'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request):
 
         connected_consoleports = ConsolePort.objects.filter(
@@ -231,7 +230,6 @@ class HomeView(View):
 
 class SearchView(View):
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request):
 
         # No query
@@ -285,7 +283,6 @@ class APIRootView(APIView):
     def get_view_name(self):
         return "API Root"
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, format=None):
 
         return Response(OrderedDict((

netbox/secrets/views.py

@@ -82,7 +82,6 @@ class SecretListView(PermissionRequiredMixin, ObjectListView):
 class SecretView(PermissionRequiredMixin, View):
     permission_required = 'secrets.view_secret'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         secret = get_object_or_404(Secret, pk=pk)

netbox/tenancy/views.py

@@ -69,7 +69,6 @@ class TenantListView(PermissionRequiredMixin, ObjectListView):
 class TenantView(PermissionRequiredMixin, View):
     permission_required = 'tenancy.view_tenant'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, slug):
 
         tenant = get_object_or_404(Tenant, slug=slug)

netbox/utilities/views.py

@@ -108,7 +108,6 @@ class ObjectListView(View):
 
         return csv_data
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request):
 
         model = self.queryset.model
@@ -716,7 +715,6 @@ class ComponentCreateView(View):
     model_form = None
     template_name = None
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         parent = get_object_or_404(self.parent_model, pk=pk)

netbox/virtualization/views.py

@@ -109,7 +109,6 @@ class ClusterListView(PermissionRequiredMixin, ObjectListView):
 class ClusterView(PermissionRequiredMixin, View):
     permission_required = 'virtualization.view_cluster'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         cluster = get_object_or_404(Cluster, pk=pk)
@@ -172,7 +171,6 @@ class ClusterAddDevicesView(PermissionRequiredMixin, View):
     form = forms.ClusterAddDevicesForm
     template_name = 'virtualization/cluster_add_devices.html'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         cluster = get_object_or_404(Cluster, pk=pk)
@@ -268,7 +266,6 @@ class VirtualMachineListView(PermissionRequiredMixin, ObjectListView):
 class VirtualMachineView(PermissionRequiredMixin, View):
     permission_required = 'virtualization.view_virtualmachine'
 
-    @method_decorator(cache_page(settings.CACHE_TIMEOUT))
     def get(self, request, pk):
 
         virtualmachine = get_object_or_404(VirtualMachine.objects.select_related('tenant__group'), pk=pk)

requirements.txt

@@ -1,9 +1,9 @@
 Django>=2.2,<2.3
+django-cacheops==4.1
 django-cors-headers==2.5.2
 django-debug-toolbar==1.11
 django-filter==2.1.0
 django-mptt==0.9.1
-django-redis==4.5.0
 django-tables2==2.0.6
 django-taggit==1.1.0
 django-taggit-serializer==0.1.7