diff --git a/netbox/dcim/models/racks.py b/netbox/dcim/models/racks.py index 79d5e892c..289c38133 100644 --- a/netbox/dcim/models/racks.py +++ b/netbox/dcim/models/racks.py @@ -18,8 +18,8 @@ from netbox.choices import ColorChoices from netbox.models import OrganizationalModel, PrimaryModel from netbox.models.features import ContactsMixin, ImageAttachmentsMixin from utilities.conversion import to_grams +from utilities.data import array_to_string, drange from utilities.fields import ColorField, NaturalOrderingField -from utilities.utils import array_to_string, drange from .device_components import PowerPort from .devices import Device, Module from .mixins import WeightMixin diff --git a/netbox/dcim/svg/racks.py b/netbox/dcim/svg/racks.py index 4779954a4..0f73095b5 100644 --- a/netbox/dcim/svg/racks.py +++ b/netbox/dcim/svg/racks.py @@ -14,8 +14,8 @@ from django.urls import reverse from django.utils.http import urlencode from netbox.config import get_config +from utilities.data import array_to_ranges from utilities.html import foreground_color -from utilities.utils import array_to_ranges from dcim.constants import RACK_ELEVATION_BORDER_WIDTH diff --git a/netbox/dcim/tests/test_models.py b/netbox/dcim/tests/test_models.py index 1a5cc8435..cab1760ed 100644 --- a/netbox/dcim/tests/test_models.py +++ b/netbox/dcim/tests/test_models.py @@ -7,7 +7,7 @@ from dcim.choices import * from dcim.models import * from extras.models import CustomField from tenancy.models import Tenant -from utilities.utils import drange +from utilities.data import drange class LocationTestCase(TestCase): diff --git a/netbox/extras/models/configs.py b/netbox/extras/models/configs.py index d0ce42b8b..6b52d4c02 100644 --- a/netbox/extras/models/configs.py +++ b/netbox/extras/models/configs.py @@ -9,11 +9,11 @@ from jinja2.sandbox import SandboxedEnvironment from extras.querysets import ConfigContextQuerySet from netbox.config import get_config -from netbox.registry import registry from netbox.models import ChangeLoggedModel from netbox.models.features import CloningMixin, CustomLinksMixin, ExportTemplatesMixin, SyncedDataMixin, TagsMixin +from netbox.registry import registry +from utilities.data import deepmerge from utilities.jinja2 import DataFileLoader -from utilities.utils import deepmerge __all__ = ( 'ConfigContext', diff --git a/netbox/extras/views.py b/netbox/extras/views.py index 553efb2c5..aa6e5718d 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -18,12 +18,13 @@ from extras.dashboard.utils import get_widget_class from netbox.constants import DEFAULT_ACTION_PERMISSIONS from netbox.views import generic from netbox.views.generic.mixins import TableMixin +from utilities.data import shallow_compare_dict from utilities.forms import ConfirmationForm, get_field_value from utilities.paginator import EnhancedPaginator, get_paginate_count from utilities.request import copy_safe_request from utilities.rqworker import get_workers_for_queue from utilities.templatetags.builtins.filters import render_markdown -from utilities.utils import count_related, normalize_querydict, shallow_compare_dict +from utilities.utils import count_related, normalize_querydict from utilities.views import ContentTypePermissionRequiredMixin, get_viewname, register_model_view from . import filtersets, forms, tables from .models import * diff --git a/netbox/ipam/models/services.py b/netbox/ipam/models/services.py index 3e3261ee9..37b559801 100644 --- a/netbox/ipam/models/services.py +++ b/netbox/ipam/models/services.py @@ -8,8 +8,7 @@ from django.utils.translation import gettext_lazy as _ from ipam.choices import * from ipam.constants import * from netbox.models import PrimaryModel -from utilities.utils import array_to_string - +from utilities.data import array_to_string __all__ = ( 'Service', diff --git a/netbox/users/api/views.py b/netbox/users/api/views.py index 412bccf59..f46cd24dc 100644 --- a/netbox/users/api/views.py +++ b/netbox/users/api/views.py @@ -14,8 +14,8 @@ from rest_framework.viewsets import ViewSet from netbox.api.viewsets import NetBoxModelViewSet from users import filtersets from users.models import Group, ObjectPermission, Token, UserConfig +from utilities.data import deepmerge from utilities.querysets import RestrictedQuerySet -from utilities.utils import deepmerge from . import serializers diff --git a/netbox/users/forms/model_forms.py b/netbox/users/forms/model_forms.py index 1f199d35c..57d22326e 100644 --- a/netbox/users/forms/model_forms.py +++ b/netbox/users/forms/model_forms.py @@ -12,11 +12,11 @@ from ipam.validators import prefix_validator from netbox.preferences import PREFERENCES from users.constants import * from users.models import * +from utilities.data import flatten_dict from utilities.forms.fields import ContentTypeMultipleChoiceField, DynamicModelMultipleChoiceField from utilities.forms.rendering import FieldSet from utilities.forms.widgets import DateTimePicker from utilities.permissions import qs_filter_from_constraints -from utilities.utils import flatten_dict __all__ = ( 'UserTokenForm', diff --git a/netbox/users/models.py b/netbox/users/models.py index d2ee16e5e..f580bd17b 100644 --- a/netbox/users/models.py +++ b/netbox/users/models.py @@ -25,8 +25,8 @@ from netaddr import IPNetwork from core.models import ObjectType from ipam.fields import IPNetworkField from netbox.config import get_config +from utilities.data import flatten_dict from utilities.querysets import RestrictedQuerySet -from utilities.utils import flatten_dict from .constants import * __all__ = ( diff --git a/netbox/users/tests/test_api.py b/netbox/users/tests/test_api.py index 2ff3545a6..7c9710f64 100644 --- a/netbox/users/tests/test_api.py +++ b/netbox/users/tests/test_api.py @@ -3,8 +3,8 @@ from django.urls import reverse from core.models import ObjectType from users.models import Group, ObjectPermission, Token +from utilities.data import deepmerge from utilities.testing import APIViewTestCases, APITestCase, create_test_user -from utilities.utils import deepmerge User = get_user_model() diff --git a/netbox/utilities/data.py b/netbox/utilities/data.py new file mode 100644 index 000000000..62eb68854 --- /dev/null +++ b/netbox/utilities/data.py @@ -0,0 +1,115 @@ +import decimal +from itertools import count, groupby + +__all__ = ( + 'array_to_ranges', + 'array_to_string', + 'deepmerge', + 'drange', + 'flatten_dict', + 'shallow_compare_dict', +) + + +# +# Dictionary utilities +# + +def deepmerge(original, new): + """ + Deep merge two dictionaries (new into original) and return a new dict + """ + merged = dict(original) + for key, val in new.items(): + if key in original and isinstance(original[key], dict) and val and isinstance(val, dict): + merged[key] = deepmerge(original[key], val) + else: + merged[key] = val + return merged + + +def flatten_dict(d, prefix='', separator='.'): + """ + Flatten nested dictionaries into a single level by joining key names with a separator. + + :param d: The dictionary to be flattened + :param prefix: Initial prefix (if any) + :param separator: The character to use when concatenating key names + """ + ret = {} + for k, v in d.items(): + key = separator.join([prefix, k]) if prefix else k + if type(v) is dict: + ret.update(flatten_dict(v, prefix=key, separator=separator)) + else: + ret[key] = v + return ret + + +def shallow_compare_dict(source_dict, destination_dict, exclude=tuple()): + """ + Return a new dictionary of the different keys. The values of `destination_dict` are returned. Only the equality of + the first layer of keys/values is checked. `exclude` is a list or tuple of keys to be ignored. + """ + difference = {} + + for key, value in destination_dict.items(): + if key in exclude: + continue + if source_dict.get(key) != value: + difference[key] = value + + return difference + + +# +# Array utilities +# + +def array_to_ranges(array): + """ + Convert an arbitrary array of integers to a list of consecutive values. Nonconsecutive values are returned as + single-item tuples. For example: + [0, 1, 2, 10, 14, 15, 16] => [(0, 2), (10,), (14, 16)]" + """ + group = ( + list(x) for _, x in groupby(sorted(array), lambda x, c=count(): next(c) - x) + ) + return [ + (g[0], g[-1])[:len(g)] for g in group + ] + + +def array_to_string(array): + """ + Generate an efficient, human-friendly string from a set of integers. Intended for use with ArrayField. + For example: + [0, 1, 2, 10, 14, 15, 16] => "0-2, 10, 14-16" + """ + ret = [] + ranges = array_to_ranges(array) + for value in ranges: + if len(value) == 1: + ret.append(str(value[0])) + else: + ret.append(f'{value[0]}-{value[1]}') + return ', '.join(ret) + + +# +# Range utilities +# + +def drange(start, end, step=decimal.Decimal(1)): + """ + Decimal-compatible implementation of Python's range() + """ + start, end, step = decimal.Decimal(start), decimal.Decimal(end), decimal.Decimal(step) + if start < end: + while start < end: + yield start + start += step + else: + while start > end: + yield start + start += step diff --git a/netbox/utilities/tests/test_utils.py b/netbox/utilities/tests/test_utils.py index 0a0c3ad2c..5bdd24108 100644 --- a/netbox/utilities/tests/test_utils.py +++ b/netbox/utilities/tests/test_utils.py @@ -1,7 +1,8 @@ from django.http import QueryDict from django.test import TestCase -from utilities.utils import deepmerge, dict_to_filter_params, normalize_querydict +from utilities.data import deepmerge +from utilities.utils import dict_to_filter_params, normalize_querydict class DictToFilterParamsTest(TestCase): diff --git a/netbox/utilities/utils.py b/netbox/utilities/utils.py index 183d78520..a7da5d542 100644 --- a/netbox/utilities/utils.py +++ b/netbox/utilities/utils.py @@ -1,5 +1,3 @@ -import decimal -from itertools import count, groupby from urllib.parse import urlencode from django.db.models import Count, ManyToOneRel, OuterRef, Subquery @@ -103,34 +101,6 @@ def normalize_querydict(querydict): } -def deepmerge(original, new): - """ - Deep merge two dictionaries (new into original) and return a new dict - """ - merged = dict(original) - for key, val in new.items(): - if key in original and isinstance(original[key], dict) and val and isinstance(val, dict): - merged[key] = deepmerge(original[key], val) - else: - merged[key] = val - return merged - - -def drange(start, end, step=decimal.Decimal(1)): - """ - Decimal-compatible implementation of Python's range() - """ - start, end, step = decimal.Decimal(start), decimal.Decimal(end), decimal.Decimal(step) - if start < end: - while start < end: - yield start - start += step - else: - while start > end: - yield start - start += step - - def prepare_cloned_fields(instance): """ Generate a QueryDict comprising attributes from an object's clone() method. @@ -154,70 +124,6 @@ def prepare_cloned_fields(instance): return QueryDict(urlencode(params), mutable=True) -def shallow_compare_dict(source_dict, destination_dict, exclude=tuple()): - """ - Return a new dictionary of the different keys. The values of `destination_dict` are returned. Only the equality of - the first layer of keys/values is checked. `exclude` is a list or tuple of keys to be ignored. - """ - difference = {} - - for key, value in destination_dict.items(): - if key in exclude: - continue - if source_dict.get(key) != value: - difference[key] = value - - return difference - - -def flatten_dict(d, prefix='', separator='.'): - """ - Flatten netsted dictionaries into a single level by joining key names with a separator. - - :param d: The dictionary to be flattened - :param prefix: Initial prefix (if any) - :param separator: The character to use when concatenating key names - """ - ret = {} - for k, v in d.items(): - key = separator.join([prefix, k]) if prefix else k - if type(v) is dict: - ret.update(flatten_dict(v, prefix=key, separator=separator)) - else: - ret[key] = v - return ret - - -def array_to_ranges(array): - """ - Convert an arbitrary array of integers to a list of consecutive values. Nonconsecutive values are returned as - single-item tuples. For example: - [0, 1, 2, 10, 14, 15, 16] => [(0, 2), (10,), (14, 16)]" - """ - group = ( - list(x) for _, x in groupby(sorted(array), lambda x, c=count(): next(c) - x) - ) - return [ - (g[0], g[-1])[:len(g)] for g in group - ] - - -def array_to_string(array): - """ - Generate an efficient, human-friendly string from a set of integers. Intended for use with ArrayField. - For example: - [0, 1, 2, 10, 14, 15, 16] => "0-2, 10, 14-16" - """ - ret = [] - ranges = array_to_ranges(array) - for value in ranges: - if len(value) == 1: - ret.append(str(value[0])) - else: - ret.append(f'{value[0]}-{value[1]}') - return ', '.join(ret) - - def content_type_name(ct, include_app=True): """ Return a human-friendly ContentType name (e.g. "DCIM > Site").