From 7c74d2ca657e3ed019abfee0230c59e3b425fd3c Mon Sep 17 00:00:00 2001 From: Jeremy Stretch Date: Fri, 7 Feb 2020 15:47:53 -0500 Subject: [PATCH] Convert interface models to use NaturalOrderingField --- netbox/dcim/managers.py | 56 +------------------ .../migrations/0096_interface_ordering.py | 53 ++++++++++++++++++ .../dcim/models/device_component_templates.py | 14 +++-- netbox/dcim/models/device_components.py | 15 +++-- netbox/utilities/ordering.py | 47 ++++++++++++++++ 5 files changed, 120 insertions(+), 65 deletions(-) create mode 100644 netbox/dcim/migrations/0096_interface_ordering.py diff --git a/netbox/dcim/managers.py b/netbox/dcim/managers.py index e1124b84e..502719646 100644 --- a/netbox/dcim/managers.py +++ b/netbox/dcim/managers.py @@ -1,18 +1,7 @@ from django.db.models import Manager, QuerySet -from django.db.models.expressions import RawSQL from .constants import NONCONNECTABLE_IFACE_TYPES -# Regular expressions for parsing Interface names -TYPE_RE = r"SUBSTRING({} FROM '^([^0-9\.:]+)')" -SLOT_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(\d{{1,9}})/') AS integer), NULL)" -SUBSLOT_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9\.:]+)?\d{{1,9}}/(\d{{1,9}})') AS integer), NULL)" -POSITION_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(?:\d{{1,9}}/){{2}}(\d{{1,9}})') AS integer), NULL)" -SUBPOSITION_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^(?:[^0-9]+)?(?:\d{{1,9}}/){{3}}(\d{{1,9}})') AS integer), NULL)" -ID_RE = r"CAST(SUBSTRING({} FROM '^(?:[^0-9\.:]+)?(\d{{1,9}})([^/]|$)') AS integer)" -CHANNEL_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^.*:(\d{{1,9}})(\.\d{{1,9}})?$') AS integer), 0)" -VC_RE = r"COALESCE(CAST(SUBSTRING({} FROM '^.*\.(\d{{1,9}})$') AS integer), 0)" - class InterfaceQuerySet(QuerySet): @@ -27,47 +16,4 @@ class InterfaceQuerySet(QuerySet): class InterfaceManager(Manager): def get_queryset(self): - """ - Naturally order interfaces by their type and numeric position. To order interfaces naturally, the `name` field - is split into eight distinct components: leading text (type), slot, subslot, position, subposition, ID, channel, - and virtual circuit: - - {type}{slot or ID}/{subslot}/{position}/{subposition}:{channel}.{vc} - - Components absent from the interface name are coalesced to zero or null. For example, an interface named - GigabitEthernet1/2/3 would be parsed as follows: - - type = 'GigabitEthernet' - slot = 1 - subslot = 2 - position = 3 - subposition = None - id = None - channel = 0 - vc = 0 - - The original `name` field is considered in its entirety to serve as a fallback in the event interfaces do not - match any of the prescribed fields. - - The `id` field is included to enforce deterministic ordering of interfaces in similar vein of other device - components. - """ - - sql_col = '{}.name'.format(self.model._meta.db_table) - ordering = [ - '_slot', '_subslot', '_position', '_subposition', '_type', '_id', '_channel', '_vc', 'name', 'pk' - - ] - - fields = { - '_type': RawSQL(TYPE_RE.format(sql_col), []), - '_id': RawSQL(ID_RE.format(sql_col), []), - '_slot': RawSQL(SLOT_RE.format(sql_col), []), - '_subslot': RawSQL(SUBSLOT_RE.format(sql_col), []), - '_position': RawSQL(POSITION_RE.format(sql_col), []), - '_subposition': RawSQL(SUBPOSITION_RE.format(sql_col), []), - '_channel': RawSQL(CHANNEL_RE.format(sql_col), []), - '_vc': RawSQL(VC_RE.format(sql_col), []), - } - - return InterfaceQuerySet(self.model, using=self._db).annotate(**fields).order_by(*ordering) + return InterfaceQuerySet(self.model, using=self._db) diff --git a/netbox/dcim/migrations/0096_interface_ordering.py b/netbox/dcim/migrations/0096_interface_ordering.py new file mode 100644 index 000000000..284066462 --- /dev/null +++ b/netbox/dcim/migrations/0096_interface_ordering.py @@ -0,0 +1,53 @@ +from django.db import migrations +import utilities.fields +import utilities.ordering + + +def _update_model_names(model): + # Update each unique field value in bulk + for name in model.objects.values_list('name', flat=True).order_by('name').distinct(): + model.objects.filter(name=name).update(_name=utilities.ordering.naturalize_interface(name)) + + +def naturalize_interfacetemplates(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'InterfaceTemplate')) + + +def naturalize_interfaces(apps, schema_editor): + _update_model_names(apps.get_model('dcim', 'Interface')) + + +class Migration(migrations.Migration): + + dependencies = [ + ('dcim', '0095_primary_model_ordering'), + ] + + operations = [ + migrations.AlterModelOptions( + name='interface', + options={'ordering': ('device', '_name')}, + ), + migrations.AlterModelOptions( + name='interfacetemplate', + options={'ordering': ('device_type', '_name')}, + ), + migrations.AddField( + model_name='interface', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize_interface), + ), + migrations.AddField( + model_name='interfacetemplate', + name='_name', + field=utilities.fields.NaturalOrderingField('target_field', blank=True, max_length=100, naturalize_function=utilities.ordering.naturalize_interface), + ), + migrations.RunPython( + code=naturalize_interfacetemplates, + reverse_code=migrations.RunPython.noop + ), + migrations.RunPython( + code=naturalize_interfaces, + reverse_code=migrations.RunPython.noop + ), + ] diff --git a/netbox/dcim/models/device_component_templates.py b/netbox/dcim/models/device_component_templates.py index 0b5c312ba..ab4a078cf 100644 --- a/netbox/dcim/models/device_component_templates.py +++ b/netbox/dcim/models/device_component_templates.py @@ -4,9 +4,9 @@ from django.db import models from dcim.choices import * from dcim.constants import * -from dcim.managers import InterfaceManager from extras.models import ObjectChange from utilities.fields import NaturalOrderingField +from utilities.ordering import naturalize_interface from utilities.utils import serialize_object from .device_components import ( ConsolePort, ConsoleServerPort, DeviceBay, FrontPort, Interface, PowerOutlet, PowerPort, RearPort, @@ -249,6 +249,12 @@ class InterfaceTemplate(ComponentTemplateModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + naturalize_function=naturalize_interface, + max_length=100, + blank=True + ) type = models.CharField( max_length=50, choices=InterfaceTypeChoices @@ -258,11 +264,9 @@ class InterfaceTemplate(ComponentTemplateModel): verbose_name='Management only' ) - objects = InterfaceManager() - class Meta: - ordering = ['device_type', 'name'] - unique_together = ['device_type', 'name'] + ordering = ('device_type', '_name') + unique_together = ('device_type', 'name') def __str__(self): return self.name diff --git a/netbox/dcim/models/device_components.py b/netbox/dcim/models/device_components.py index 3eb9ac74e..a41eda576 100644 --- a/netbox/dcim/models/device_components.py +++ b/netbox/dcim/models/device_components.py @@ -10,9 +10,9 @@ from dcim.choices import * from dcim.constants import * from dcim.exceptions import LoopDetected from dcim.fields import MACAddressField -from dcim.managers import InterfaceManager from extras.models import ObjectChange, TaggedItem from utilities.fields import NaturalOrderingField +from utilities.ordering import naturalize_interface from utilities.utils import serialize_object from virtualization.choices import VMInterfaceTypeChoices @@ -529,6 +529,12 @@ class Interface(CableTermination, ComponentModel): name = models.CharField( max_length=64 ) + _name = NaturalOrderingField( + target_field='name', + naturalize_function=naturalize_interface, + max_length=100, + blank=True + ) _connected_interface = models.OneToOneField( to='self', on_delete=models.SET_NULL, @@ -597,8 +603,6 @@ class Interface(CableTermination, ComponentModel): blank=True, verbose_name='Tagged VLANs' ) - - objects = InterfaceManager() tags = TaggableManager(through=TaggedItem) csv_headers = [ @@ -607,8 +611,9 @@ class Interface(CableTermination, ComponentModel): ] class Meta: - ordering = ['device', 'name'] - unique_together = ['device', 'name'] + # TODO: ordering and unique_together should include virtual_machine + ordering = ('device', '_name') + unique_together = ('device', 'name') def __str__(self): return self.name diff --git a/netbox/utilities/ordering.py b/netbox/utilities/ordering.py index 88a46d3d3..d459e6f6c 100644 --- a/netbox/utilities/ordering.py +++ b/netbox/utilities/ordering.py @@ -1,5 +1,14 @@ import re +INTERFACE_NAME_REGEX = r'(^(?P[^\d\.:]+)?)' \ + r'((?P\d+)/)?' \ + r'((?P\d+)/)?' \ + r'((?P\d+)/)?' \ + r'((?P\d+)/)?' \ + r'((?P\d+))?' \ + r'(:(?P\d+))?' \ + r'(.(?P\d+)$)?' + def naturalize(value, max_length=None, integer_places=8): """ @@ -31,3 +40,41 @@ def naturalize(value, max_length=None, integer_places=8): ret = ''.join(output) return ret[:max_length] if max_length else ret + + +def naturalize_interface(value, max_length=None): + """ + Similar in nature to naturalize(), but takes into account a particular naming format adapted from the old + InterfaceManager. + + :param value: The value to be naturalized + :param max_length: The maximum length of the returned string. Characters beyond this length will be stripped. + """ + output = [] + match = re.search(INTERFACE_NAME_REGEX, value) + if match is None: + return value + + # First, we order by slot/position, padding each to four digits. If a field is not present, + # set it to 9999 to ensure it is ordered last. + for part_name in ('slot', 'subslot', 'position', 'subposition'): + part = match.group(part_name) + if part is not None: + output.append(part.rjust(4, '0')) + else: + output.append('9999') + + # Append the type, if any. + if match.group('type') is not None: + output.append(match.group('type')) + + # Finally, append any remaining fields, left-padding to eight digits each. + for part_name in ('id', 'channel', 'vc'): + part = match.group(part_name) + if part is not None: + output.append(part.rjust(6, '0')) + else: + output.append('000000') + + ret = ''.join(output) + return ret[:max_length] if max_length else ret