Moved querysets to separate files; tweaked interface ordering logic (#1523)

netbox/dcim/models.py

@@ -13,7 +13,6 @@ from django.core.exceptions import ValidationError
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
 from django.db.models import Count, Q, ObjectDoesNotExist
-from django.db.models.expressions import RawSQL
 from django.urls import reverse
 from django.utils.encoding import python_2_unicode_compatible
 
@@ -27,6 +26,7 @@ from utilities.models import CreatedUpdatedModel
 from utilities.utils import csv_format
 from .constants import *
 from .fields import ASNField, MACAddressField
+from .querysets import InterfaceQuerySet
 
 
 #
@@ -687,72 +687,6 @@ class PowerOutletTemplate(models.Model):
         return self.name
 
 
-class InterfaceQuerySet(models.QuerySet):
-
-    def order_naturally(self, method=IFACE_ORDERING_POSITION):
-        """
-        Naturally order interfaces by their type and numeric position. The sort method must be one of the defined
-        IFACE_ORDERING_CHOICES (typically indicated by a parent Device's DeviceType).
-
-        To order interfaces naturally, the `name` field is split into six distinct components: leading text (type),
-        slot, subslot, position, channel, and virtual circuit:
-
-            {type}{slot}/{subslot}/{position}/{subposition}:{channel}.{vc}
-
-        Components absent from the interface name are ignored. For example, an interface named GigabitEthernet1/2/3
-        would be parsed as follows:
-
-            name = 'GigabitEthernet'
-            slot =  1
-            subslot = 2
-            position = 3
-            subposition = 0
-            channel = None
-            vc = 0
-
-        The original `name` field is taken as a whole to serve as a fallback in the event interfaces do not match any of
-        the prescribed fields.
-        """
-        sql_col = '{}.name'.format(self.model._meta.db_table)
-        ordering = {
-            IFACE_ORDERING_POSITION: (
-                '_slot', '_subslot', '_position', '_subposition', '_channel', '_vc', '_type', '_id', 'name',
-            ),
-            IFACE_ORDERING_NAME: (
-                '_type', '_slot', '_subslot', '_position', '_subposition', '_channel', '_vc', '_id', 'name',
-            ),
-        }[method]
-
-        TYPE_RE = r"SUBSTRING({} FROM '^([^0-9]+)')"
-        ID_RE = r"CAST(SUBSTRING({} FROM '^(?:[^0-9]+)([0-9]+)$') AS integer)"
-        SLOT_RE = r"CAST(SUBSTRING({} FROM '^(?:[^0-9]+)([0-9]+)\/') AS integer)"
-        SUBSLOT_RE = r"CAST(SUBSTRING({} FROM '^(?:[^0-9]+)(?:[0-9]+\/)([0-9]+)') AS integer)"
-        POSITION_RE = r"CAST(SUBSTRING({} FROM '^(?:[^0-9]+)(?:[0-9]+\/){{2}}([0-9]+)') AS integer)"
-        SUBPOSITION_RE = r"CAST(SUBSTRING({} FROM '^(?:[^0-9]+)(?:[0-9]+\/){{3}}([0-9]+)') AS integer)"
-        CHANNEL_RE = r"COALESCE(CAST(SUBSTRING({} FROM ':([0-9]+)(\.[0-9]+)?$') AS integer), 0)"
-        VC_RE = r"COALESCE(CAST(SUBSTRING({} FROM '\.([0-9]+)$') AS integer), 0)"
-
-        fields = {
-            '_type': RawSQL(TYPE_RE.format(sql_col), []),
-            '_id': RawSQL(ID_RE.format(sql_col), []),
-            '_slot': RawSQL(SLOT_RE.format(sql_col), []),
-            '_subslot': RawSQL(SUBSLOT_RE.format(sql_col), []),
-            '_position': RawSQL(POSITION_RE.format(sql_col), []),
-            '_subposition': RawSQL(SUBPOSITION_RE.format(sql_col), []),
-            '_channel': RawSQL(CHANNEL_RE.format(sql_col), []),
-            '_vc': RawSQL(VC_RE.format(sql_col), []),
-        }
-
-        return self.annotate(**fields).order_by(*ordering)
-
-    def connectable(self):
-        """
-        Return only physical interfaces which are capable of being connected to other interfaces (i.e. not virtual or
-        wireless).
-        """
-        return self.exclude(form_factor__in=NONCONNECTABLE_IFACE_TYPES)
-
-
 @python_2_unicode_compatible
 class InterfaceTemplate(models.Model):
     """

netbox/dcim/querysets.py

@@ -0,0 +1,72 @@
+from __future__ import unicode_literals
+
+from django.db.models import QuerySet
+from django.db.models.expressions import RawSQL
+
+from .constants import IFACE_ORDERING_NAME, IFACE_ORDERING_POSITION, NONCONNECTABLE_IFACE_TYPES
+
+
+class InterfaceQuerySet(QuerySet):
+
+    def order_naturally(self, method=IFACE_ORDERING_POSITION):
+        """
+        Naturally order interfaces by their type and numeric position. The sort method must be one of the defined
+        IFACE_ORDERING_CHOICES (typically indicated by a parent Device's DeviceType).
+
+        To order interfaces naturally, the `name` field is split into six distinct components: leading text (type),
+        slot, subslot, position, channel, and virtual circuit:
+
+            {type}{slot}/{subslot}/{position}/{subposition}:{channel}.{vc}
+
+        Components absent from the interface name are ignored. For example, an interface named GigabitEthernet1/2/3
+        would be parsed as follows:
+
+            name = 'GigabitEthernet'
+            slot =  1
+            subslot = 2
+            position = 3
+            subposition = 0
+            channel = None
+            vc = 0
+
+        The original `name` field is taken as a whole to serve as a fallback in the event interfaces do not match any of
+        the prescribed fields.
+        """
+        sql_col = '{}.name'.format(self.model._meta.db_table)
+        ordering = {
+            IFACE_ORDERING_POSITION: (
+                '_slot', '_subslot', '_position', '_subposition', '_channel', '_type', '_vc', '_id', 'name',
+            ),
+            IFACE_ORDERING_NAME: (
+                '_type', '_slot', '_subslot', '_position', '_subposition', '_channel', '_vc', '_id', 'name',
+            ),
+        }[method]
+
+        TYPE_RE = r"SUBSTRING({} FROM '^([^0-9]+)')"
+        ID_RE = r"CAST(SUBSTRING({} FROM '^(?:[^0-9]+)([0-9]+)$') AS integer)"
+        SLOT_RE = r"CAST(SUBSTRING({} FROM '^(?:[^0-9]+)([0-9]+)\/') AS integer)"
+        SUBSLOT_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)(?:[0-9]+\/)([0-9]+)') AS integer), 0)"
+        POSITION_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)(?:[0-9]+\/){{2}}([0-9]+)') AS integer), 0)"
+        SUBPOSITION_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)(?:[0-9]+\/){{3}}([0-9]+)') AS integer), 0)"
+        CHANNEL_RE = r"COALESCE(CAST(SUBSTRING({} FROM ':([0-9]+)(\.[0-9]+)?$') AS integer), 0)"
+        VC_RE = r"COALESCE(CAST(SUBSTRING({} FROM '\.([0-9]+)$') AS integer), 0)"
+
+        fields = {
+            '_type': RawSQL(TYPE_RE.format(sql_col), []),
+            '_id': RawSQL(ID_RE.format(sql_col), []),
+            '_slot': RawSQL(SLOT_RE.format(sql_col), []),
+            '_subslot': RawSQL(SUBSLOT_RE.format(sql_col), []),
+            '_position': RawSQL(POSITION_RE.format(sql_col), []),
+            '_subposition': RawSQL(SUBPOSITION_RE.format(sql_col), []),
+            '_channel': RawSQL(CHANNEL_RE.format(sql_col), []),
+            '_vc': RawSQL(VC_RE.format(sql_col), []),
+        }
+
+        return self.annotate(**fields).order_by(*ordering)
+
+    def connectable(self):
+        """
+        Return only physical interfaces which are capable of being connected to other interfaces (i.e. not virtual or
+        wireless).
+        """
+        return self.exclude(form_factor__in=NONCONNECTABLE_IFACE_TYPES)

netbox/ipam/models.py

@@ -2,12 +2,11 @@ from __future__ import unicode_literals
 import netaddr
 
 from django.conf import settings
-from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation
+from django.contrib.contenttypes.fields import GenericRelation
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
 from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
-from django.db.models import Q
 from django.db.models.expressions import RawSQL
 from django.urls import reverse
 from django.utils.encoding import python_2_unicode_compatible
@@ -16,10 +15,10 @@ from dcim.models import Interface
 from extras.models import CustomFieldModel, CustomFieldValue
 from tenancy.models import Tenant
 from utilities.models import CreatedUpdatedModel
-from utilities.sql import NullsFirstQuerySet
 from utilities.utils import csv_format
 from .constants import *
 from .fields import IPNetworkField, IPAddressField
+from .querysets import PrefixQuerySet
 
 
 @python_2_unicode_compatible
@@ -190,41 +189,6 @@ class Role(models.Model):
         return self.vlans.count()
 
 
-class PrefixQuerySet(NullsFirstQuerySet):
-
-    def annotate_depth(self, limit=None):
-        """
-        Iterate through a QuerySet of Prefixes and annotate the hierarchical level of each. While it would be preferable
-        to do this using .extra() on the QuerySet to count the unique parents of each prefix, that approach introduces
-        performance issues at scale.
-
-        Because we're adding a non-field attribute to the model, annotation must be made *after* any QuerySet
-        modifications.
-        """
-        queryset = self
-        stack = []
-        for p in queryset:
-            try:
-                prev_p = stack[-1]
-            except IndexError:
-                prev_p = None
-            if prev_p is not None:
-                while (p.prefix not in prev_p.prefix) or p.prefix == prev_p.prefix:
-                    stack.pop()
-                    try:
-                        prev_p = stack[-1]
-                    except IndexError:
-                        prev_p = None
-                        break
-            if prev_p is not None:
-                prev_p.has_children = True
-            stack.append(p)
-            p.depth = len(stack) - 1
-        if limit is None:
-            return queryset
-        return list(filter(lambda p: p.depth <= limit, queryset))
-
-
 @python_2_unicode_compatible
 class Prefix(CreatedUpdatedModel, CustomFieldModel):
     """

netbox/ipam/querysets.py

@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+
+from utilities.sql import NullsFirstQuerySet
+
+
+class PrefixQuerySet(NullsFirstQuerySet):
+
+    def annotate_depth(self, limit=None):
+        """
+        Iterate through a QuerySet of Prefixes and annotate the hierarchical level of each. While it would be preferable
+        to do this using .extra() on the QuerySet to count the unique parents of each prefix, that approach introduces
+        performance issues at scale.
+
+        Because we're adding a non-field attribute to the model, annotation must be made *after* any QuerySet
+        modifications.
+        """
+        queryset = self
+        stack = []
+        for p in queryset:
+            try:
+                prev_p = stack[-1]
+            except IndexError:
+                prev_p = None
+            if prev_p is not None:
+                while (p.prefix not in prev_p.prefix) or p.prefix == prev_p.prefix:
+                    stack.pop()
+                    try:
+                        prev_p = stack[-1]
+                    except IndexError:
+                        prev_p = None
+                        break
+            if prev_p is not None:
+                prev_p.has_children = True
+            stack.append(p)
+            p.depth = len(stack) - 1
+        if limit is None:
+            return queryset
+        return list(filter(lambda p: p.depth <= limit, queryset))

netbox/secrets/models.py

@@ -17,6 +17,7 @@ from dcim.models import Device
 from utilities.models import CreatedUpdatedModel
 from .exceptions import InvalidKey
 from .hashers import SecretValidationHasher
+from .querysets import UserKeyQuerySet
 
 
 def generate_random_key(bits=256):
@@ -46,16 +47,6 @@ def decrypt_master_key(master_key_cipher, private_key):
     return cipher.decrypt(master_key_cipher)
 
 
-class UserKeyQuerySet(models.QuerySet):
-
-    def active(self):
-        return self.filter(master_key_cipher__isnull=False)
-
-    def delete(self):
-        # Disable bulk deletion to avoid accidentally wiping out all copies of the master key.
-        raise Exception("Bulk deletion has been disabled.")
-
-
 @python_2_unicode_compatible
 class UserKey(CreatedUpdatedModel):
     """

netbox/secrets/querysets.py

@@ -0,0 +1,13 @@
+from __future__ import unicode_literals
+
+from django.db.models import QuerySet
+
+
+class UserKeyQuerySet(QuerySet):
+
+    def active(self):
+        return self.filter(master_key_cipher__isnull=False)
+
+    def delete(self):
+        # Disable bulk deletion to avoid accidentally wiping out all copies of the master key.
+        raise Exception("Bulk deletion has been disabled.")