From ffae2c5f184f5cc2afc20076148c6933bfd1a528 Mon Sep 17 00:00:00 2001 From: WillIrvine Date: Fri, 23 Jul 2021 11:08:41 +1200 Subject: [PATCH 01/15] Fixes #6632 --- netbox/ipam/lookups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox/ipam/lookups.py b/netbox/ipam/lookups.py index 8d9071dee..c6abb5a26 100644 --- a/netbox/ipam/lookups.py +++ b/netbox/ipam/lookups.py @@ -151,7 +151,7 @@ class NetHostContained(Lookup): lhs, lhs_params = self.process_lhs(qn, connection) rhs, rhs_params = self.process_rhs(qn, connection) params = lhs_params + rhs_params - return 'CAST(HOST(%s) AS INET) << %s' % (lhs, rhs), params + return 'CAST(HOST(%s) AS INET) <<= %s' % (lhs, rhs), params class NetFamily(Transform): From fce419526daf5c975794e43d7b042bcae00aea9b Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 13 Aug 2021 15:26:06 -0400 Subject: [PATCH 02/15] Closes #6748: Add site group filter to devices list --- docs/release-notes/version-2.11.md | 8 ++++++++ netbox/dcim/forms.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 40df841c0..bab219ca5 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -1,5 +1,13 @@ # NetBox v2.11 +## v2.11.12 (FUTURE) + +### Enhancements + +* [#6748](https://github.com/netbox-community/netbox/issues/6748) - Add site group filter to devices list + +--- + ## v2.11.11 (2021-08-12) ### Enhancements diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 97f8721a5..81782f8ee 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -2421,8 +2421,8 @@ class DeviceBulkEditForm(BootstrapMixin, AddRemoveTagsForm, CustomFieldBulkEditF class DeviceFilterForm(BootstrapMixin, LocalConfigContextFilterForm, TenancyFilterForm, CustomFieldFilterForm): model = Device field_order = [ - 'q', 'region_id', 'site_id', 'location_id', 'rack_id', 'status', 'role_id', 'tenant_group_id', 'tenant_id', - 'manufacturer_id', 'device_type_id', 'asset_tag', 'mac_address', 'has_primary_ip', + 'q', 'region_id', 'site_group_id', 'site_id', 'location_id', 'rack_id', 'status', 'role_id', 'tenant_group_id', + 'tenant_id', 'manufacturer_id', 'device_type_id', 'asset_tag', 'mac_address', 'has_primary_ip', ] q = forms.CharField( required=False, @@ -2433,11 +2433,17 @@ class DeviceFilterForm(BootstrapMixin, LocalConfigContextFilterForm, TenancyFilt required=False, label=_('Region') ) + site_group_id = DynamicModelMultipleChoiceField( + queryset=SiteGroup.objects.all(), + required=False, + label=_('Site group') + ) site_id = DynamicModelMultipleChoiceField( queryset=Site.objects.all(), required=False, query_params={ - 'region_id': '$region_id' + 'region_id': '$region_id', + 'group_id': '$site_group_id', }, label=_('Site') ) From 3feba2997f989241862c37d353144acd734d2208 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 13 Aug 2021 15:56:14 -0400 Subject: [PATCH 03/15] Closes #6872: Add table configuration button to child prefixes view --- docs/release-notes/version-2.11.md | 1 + netbox/ipam/views.py | 4 +++- netbox/templates/ipam/prefix/prefixes.html | 10 ++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index bab219ca5..77bd33dbf 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -5,6 +5,7 @@ ### Enhancements * [#6748](https://github.com/netbox-community/netbox/issues/6748) - Add site group filter to devices list +* [#6872](https://github.com/netbox-community/netbox/issues/6872) - Add table configuration button to child prefixes view --- diff --git a/netbox/ipam/views.py b/netbox/ipam/views.py index 95546fcc6..33d332d40 100644 --- a/netbox/ipam/views.py +++ b/netbox/ipam/views.py @@ -4,6 +4,7 @@ from django.shortcuts import get_object_or_404, redirect, render from dcim.models import Device, Interface from netbox.views import generic +from utilities.forms import TableConfigForm from utilities.tables import paginate_table from utilities.utils import count_related from virtualization.models import VirtualMachine, VMInterface @@ -412,7 +413,7 @@ class PrefixPrefixesView(generic.ObjectView): if child_prefixes and request.GET.get('show_available', 'true') == 'true': child_prefixes = add_available_prefixes(instance.prefix, child_prefixes) - prefix_table = tables.PrefixDetailTable(child_prefixes) + prefix_table = tables.PrefixDetailTable(child_prefixes, user=request.user) if request.user.has_perm('ipam.change_prefix') or request.user.has_perm('ipam.delete_prefix'): prefix_table.columns.show('pk') paginate_table(prefix_table, request) @@ -433,6 +434,7 @@ class PrefixPrefixesView(generic.ObjectView): 'bulk_querystring': bulk_querystring, 'active_tab': 'prefixes', 'show_available': request.GET.get('show_available', 'true') == 'true', + 'table_config_form': TableConfigForm(table=prefix_table), } diff --git a/netbox/templates/ipam/prefix/prefixes.html b/netbox/templates/ipam/prefix/prefixes.html index 61baa2f1e..e9e3acd77 100644 --- a/netbox/templates/ipam/prefix/prefixes.html +++ b/netbox/templates/ipam/prefix/prefixes.html @@ -1,7 +1,12 @@ {% extends 'ipam/prefix/base.html' %} +{% load helpers %} +{% load static %} {% block buttons %} {% include 'ipam/inc/toggle_available.html' %} + {% if request.user.is_authenticated and table_config_form %} + + {% endif %} {% if perms.ipam.add_prefix and active_tab == 'prefixes' and first_available_prefix %} Add Child Prefix @@ -22,4 +27,9 @@ {% include 'utilities/obj_table.html' with table=prefix_table table_template='panel_table.html' heading='Child Prefixes' bulk_edit_url='ipam:prefix_bulk_edit' bulk_delete_url='ipam:prefix_bulk_delete' parent=prefix %} + {% table_config_form prefix_table table_name="PrefixDetailTable" %} +{% endblock %} + +{% block javascript %} + {% endblock %} From 5a8cedd63f6ce172d4a7412f953d09c1ef79c2bd Mon Sep 17 00:00:00 2001 From: bluikko <14869000+bluikko@users.noreply.github.com> Date: Mon, 16 Aug 2021 11:30:13 +0700 Subject: [PATCH 04/15] Add hardwired PowerOutlet --- netbox/dcim/choices.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/netbox/dcim/choices.py b/netbox/dcim/choices.py index 4216d9355..63054c2ce 100644 --- a/netbox/dcim/choices.py +++ b/netbox/dcim/choices.py @@ -553,6 +553,8 @@ class PowerOutletTypeChoices(ChoiceSet): # Proprietary TYPE_HDOT_CX = 'hdot-cx' TYPE_SAF_D_GRID = 'saf-d-grid' + # Other + TYPE_HARDWIRED = 'hardwired' CHOICES = ( ('IEC 60320', ( @@ -654,6 +656,9 @@ class PowerOutletTypeChoices(ChoiceSet): (TYPE_HDOT_CX, 'HDOT Cx'), (TYPE_SAF_D_GRID, 'Saf-D-Grid'), )), + ('Other', ( + (TYPE_HARDWIRED, 'Hardwired'), + )), ) From 5b89cdc86894f6db5d55d686324c7bcde9b3a088 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 16 Aug 2021 13:41:15 -0400 Subject: [PATCH 05/15] Fixes #5968: Model forms should save empty custom field values as null --- docs/release-notes/version-2.11.md | 4 +++ netbox/extras/forms.py | 6 +++- netbox/extras/tests/test_forms.py | 53 ++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 netbox/extras/tests/test_forms.py diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 77bd33dbf..1379a9de1 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -7,6 +7,10 @@ * [#6748](https://github.com/netbox-community/netbox/issues/6748) - Add site group filter to devices list * [#6872](https://github.com/netbox-community/netbox/issues/6872) - Add table configuration button to child prefixes view +### Bug Fixes + +* [#5968](https://github.com/netbox-community/netbox/issues/5968) - Model forms should save empty custom field values as null + --- ## v2.11.11 (2021-08-12) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index ab1c5aded..25fc7813d 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -77,7 +77,11 @@ class CustomFieldModelForm(forms.ModelForm): # Save custom field data on instance for cf_name in self.custom_fields: - self.instance.custom_field_data[cf_name[3:]] = self.cleaned_data.get(cf_name) + key = cf_name[3:] # Strip "cf_" from field name + value = self.cleaned_data.get(cf_name) + empty_values = self.fields[cf_name].empty_values + # Convert "empty" values to null + self.instance.custom_field_data[key] = value if value not in empty_values else None return super().clean() diff --git a/netbox/extras/tests/test_forms.py b/netbox/extras/tests/test_forms.py new file mode 100644 index 000000000..cb0a9c081 --- /dev/null +++ b/netbox/extras/tests/test_forms.py @@ -0,0 +1,53 @@ +from django.contrib.contenttypes.models import ContentType +from django.test import TestCase + +from dcim.forms import SiteForm +from dcim.models import Site +from extras.choices import CustomFieldTypeChoices +from extras.models import CustomField + + +class CustomFieldModelFormTest(TestCase): + + @classmethod + def setUpTestData(cls): + obj_type = ContentType.objects.get_for_model(Site) + CHOICES = ('A', 'B', 'C') + + cf_text = CustomField.objects.create(name='text', type=CustomFieldTypeChoices.TYPE_TEXT) + cf_text.content_types.set([obj_type]) + + cf_integer = CustomField.objects.create(name='integer', type=CustomFieldTypeChoices.TYPE_INTEGER) + cf_integer.content_types.set([obj_type]) + + cf_boolean = CustomField.objects.create(name='boolean', type=CustomFieldTypeChoices.TYPE_BOOLEAN) + cf_boolean.content_types.set([obj_type]) + + cf_date = CustomField.objects.create(name='date', type=CustomFieldTypeChoices.TYPE_DATE) + cf_date.content_types.set([obj_type]) + + cf_url = CustomField.objects.create(name='url', type=CustomFieldTypeChoices.TYPE_URL) + cf_url.content_types.set([obj_type]) + + cf_select = CustomField.objects.create(name='select', type=CustomFieldTypeChoices.TYPE_SELECT, choices=CHOICES) + cf_select.content_types.set([obj_type]) + + cf_multiselect = CustomField.objects.create(name='multiselect', type=CustomFieldTypeChoices.TYPE_MULTISELECT, + choices=CHOICES) + cf_multiselect.content_types.set([obj_type]) + + def test_empty_values(self): + """ + Test that empty custom field values are stored as null + """ + form = SiteForm({ + 'name': 'Site 1', + 'slug': 'site-1', + 'status': 'active', + }) + self.assertTrue(form.is_valid()) + instance = form.save() + + for field_type, _ in CustomFieldTypeChoices.CHOICES: + self.assertIn(field_type, instance.custom_field_data) + self.assertIsNone(instance.custom_field_data[field_type]) From 9b0258fef4131461a7c82c60c99e5259df9c7b27 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 16 Aug 2021 14:38:06 -0400 Subject: [PATCH 06/15] Fixes #6686: Force assignment of null custom field values to objects --- docs/release-notes/version-2.11.md | 1 + netbox/extras/models/customfields.py | 27 +++++++++++++++++------- netbox/extras/signals.py | 11 +++++++++- netbox/extras/tests/test_customfields.py | 10 +++++++-- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 1379a9de1..a3061d866 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -10,6 +10,7 @@ ### Bug Fixes * [#5968](https://github.com/netbox-community/netbox/issues/5968) - Model forms should save empty custom field values as null +* [#6686](https://github.com/netbox-community/netbox/issues/6686) - Force assignment of null custom field values to objects --- diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index a433a3f81..089621f92 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -120,17 +120,16 @@ class CustomField(BigIDModel): # Cache instance's original name so we can check later whether it has changed self._name = self.name - def rename_object_data(self, old_name, new_name): + def populate_initial_data(self, content_types): """ - Called when a CustomField has been renamed. Updates all assigned object data. + Populate initial custom field data upon either a) the creation of a new CustomField, or + b) the assignment of an existing CustomField to new object types. """ - for ct in self.content_types.all(): + for ct in content_types: model = ct.model_class() - params = {f'custom_field_data__{old_name}__isnull': False} - instances = model.objects.filter(**params) - for instance in instances: - instance.custom_field_data[new_name] = instance.custom_field_data.pop(old_name) - model.objects.bulk_update(instances, ['custom_field_data'], batch_size=100) + for obj in model.objects.exclude(**{f'custom_field_data__contains': self.name}): + obj.custom_field_data[self.name] = self.default + obj.save() def remove_stale_data(self, content_types): """ @@ -143,6 +142,18 @@ class CustomField(BigIDModel): del(obj.custom_field_data[self.name]) obj.save() + def rename_object_data(self, old_name, new_name): + """ + Called when a CustomField has been renamed. Updates all assigned object data. + """ + for ct in self.content_types.all(): + model = ct.model_class() + params = {f'custom_field_data__{old_name}__isnull': False} + instances = model.objects.filter(**params) + for instance in instances: + instance.custom_field_data[new_name] = instance.custom_field_data.pop(old_name) + model.objects.bulk_update(instances, ['custom_field_data'], batch_size=100) + def clean(self): super().clean() diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py index 2fc292294..e68b19a89 100644 --- a/netbox/extras/signals.py +++ b/netbox/extras/signals.py @@ -108,6 +108,14 @@ def _handle_deleted_object(request, webhook_queue, sender, instance, **kwargs): # Custom fields # +def handle_cf_added_obj_types(instance, action, pk_set, **kwargs): + """ + Handle the population of default/null values when a CustomField is added to one or more ContentTypes. + """ + if action == 'post_add': + instance.populate_initial_data(ContentType.objects.filter(pk__in=pk_set)) + + def handle_cf_removed_obj_types(instance, action, pk_set, **kwargs): """ Handle the cleanup of old custom field data when a CustomField is removed from one or more ContentTypes. @@ -131,9 +139,10 @@ def handle_cf_deleted(instance, **kwargs): instance.remove_stale_data(instance.content_types.all()) -m2m_changed.connect(handle_cf_removed_obj_types, sender=CustomField.content_types.through) post_save.connect(handle_cf_renamed, sender=CustomField) pre_delete.connect(handle_cf_deleted, sender=CustomField) +m2m_changed.connect(handle_cf_added_obj_types, sender=CustomField.content_types.through) +m2m_changed.connect(handle_cf_removed_obj_types, sender=CustomField.content_types.through) # diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index c14424ba6..b1bf10be6 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -42,8 +42,11 @@ class CustomFieldTest(TestCase): cf.save() cf.content_types.set([obj_type]) - # Assign a value to the first Site + # Check that the field has a null initial value site = Site.objects.first() + self.assertIsNone(site.custom_field_data[cf.name]) + + # Assign a value to the first Site site.custom_field_data[cf.name] = data['field_value'] site.save() @@ -73,8 +76,11 @@ class CustomFieldTest(TestCase): cf.save() cf.content_types.set([obj_type]) - # Assign a value to the first Site + # Check that the field has a null initial value site = Site.objects.first() + self.assertIsNone(site.custom_field_data[cf.name]) + + # Assign a value to the first Site site.custom_field_data[cf.name] = 'Option A' site.save() From 10847e2956be50ea62f476fa68f979178fae3260 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 16 Aug 2021 14:48:56 -0400 Subject: [PATCH 07/15] Optimize addition/removal of default custom field values --- netbox/extras/models/customfields.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/netbox/extras/models/customfields.py b/netbox/extras/models/customfields.py index 089621f92..ec1f042d2 100644 --- a/netbox/extras/models/customfields.py +++ b/netbox/extras/models/customfields.py @@ -127,9 +127,10 @@ class CustomField(BigIDModel): """ for ct in content_types: model = ct.model_class() - for obj in model.objects.exclude(**{f'custom_field_data__contains': self.name}): - obj.custom_field_data[self.name] = self.default - obj.save() + instances = model.objects.exclude(**{f'custom_field_data__contains': self.name}) + for instance in instances: + instance.custom_field_data[self.name] = self.default + model.objects.bulk_update(instances, ['custom_field_data'], batch_size=100) def remove_stale_data(self, content_types): """ @@ -138,9 +139,10 @@ class CustomField(BigIDModel): """ for ct in content_types: model = ct.model_class() - for obj in model.objects.filter(**{f'custom_field_data__{self.name}__isnull': False}): - del(obj.custom_field_data[self.name]) - obj.save() + instances = model.objects.filter(**{f'custom_field_data__{self.name}__isnull': False}) + for instance in instances: + del(instance.custom_field_data[self.name]) + model.objects.bulk_update(instances, ['custom_field_data'], batch_size=100) def rename_object_data(self, old_name, new_name): """ From d850aa07733d6042553817c14ebfe6c4acb88dd8 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 20 Aug 2021 09:17:53 -0400 Subject: [PATCH 08/15] Changelog for #6790 --- docs/release-notes/version-2.11.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index a3061d866..6eb04d9b4 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -5,6 +5,7 @@ ### Enhancements * [#6748](https://github.com/netbox-community/netbox/issues/6748) - Add site group filter to devices list +* [#6790](https://github.com/netbox-community/netbox/issues/6790) - Recognize a /32 IPv4 address as a child of a /32 IPv4 prefix * [#6872](https://github.com/netbox-community/netbox/issues/6872) - Add table configuration button to child prefixes view ### Bug Fixes From 53a5bc2221205f3610628c7e9cb5abb806954936 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 20 Aug 2021 16:06:37 -0400 Subject: [PATCH 09/15] Fixes #6929: Introduce LOGIN_PERSISTENCE configuration parameter to persist user sessions --- docs/configuration/optional-settings.md | 10 ++++++++++ docs/release-notes/version-2.11.md | 1 + netbox/netbox/configuration.example.py | 4 ++++ netbox/netbox/settings.py | 2 ++ 4 files changed, 17 insertions(+) diff --git a/docs/configuration/optional-settings.md b/docs/configuration/optional-settings.md index 6c62fc6d1..bde911a0e 100644 --- a/docs/configuration/optional-settings.md +++ b/docs/configuration/optional-settings.md @@ -257,6 +257,16 @@ LOGGING = { --- +## LOGIN_PERSISTENCE + +Default: False + +If true, the lifetime of a user's authentication session will be automatically reset upon each valid request. For example, if [`LOGIN_TIMEOUT`](#login_timeout) is configured to 14 days (the default), and a user whose session is due to expire in five days makes a NetBox request (with a valid session cookie), the session's lifetime will be reset to 14 days. + +Note that enabling this setting causes NetBox to update a user's session in the database (or file, as configured per [`SESSION_FILE_PATH`](#session_file_path)) with each request, which may introduce significant overhead in very active environments. It also permits an active user to remain authenticated to NetBox indefinitely. + +--- + ## LOGIN_REQUIRED Default: False diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 6eb04d9b4..8db5ccd05 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -7,6 +7,7 @@ * [#6748](https://github.com/netbox-community/netbox/issues/6748) - Add site group filter to devices list * [#6790](https://github.com/netbox-community/netbox/issues/6790) - Recognize a /32 IPv4 address as a child of a /32 IPv4 prefix * [#6872](https://github.com/netbox-community/netbox/issues/6872) - Add table configuration button to child prefixes view +* [#6929](https://github.com/netbox-community/netbox/issues/6929) - Introduce `LOGIN_PERSISTENCE` configuration parameter to persist user sessions ### Bug Fixes diff --git a/netbox/netbox/configuration.example.py b/netbox/netbox/configuration.example.py index cac4a9c85..231c2513e 100644 --- a/netbox/netbox/configuration.example.py +++ b/netbox/netbox/configuration.example.py @@ -149,6 +149,10 @@ INTERNAL_IPS = ('127.0.0.1', '::1') # https://docs.djangoproject.com/en/stable/topics/logging/ LOGGING = {} +# Automatically reset the lifetime of a valid session upon each authenticated request. Enables users to remain +# authenticated to NetBox indefinitely. +LOGIN_PERSISTENCE = False + # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users # are permitted to access most data in NetBox (excluding secrets) but not make any changes. LOGIN_REQUIRED = False diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 1f3cf8049..c3e19d3a2 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -103,6 +103,7 @@ NAPALM_PASSWORD = getattr(configuration, 'NAPALM_PASSWORD', '') NAPALM_TIMEOUT = getattr(configuration, 'NAPALM_TIMEOUT', 30) NAPALM_USERNAME = getattr(configuration, 'NAPALM_USERNAME', '') PAGINATE_COUNT = getattr(configuration, 'PAGINATE_COUNT', 50) +LOGIN_PERSISTENCE = getattr(configuration, 'LOGIN_PERSISTENCE', False) PLUGINS = getattr(configuration, 'PLUGINS', []) PLUGINS_CONFIG = getattr(configuration, 'PLUGINS_CONFIG', {}) PREFER_IPV4 = getattr(configuration, 'PREFER_IPV4', False) @@ -251,6 +252,7 @@ CACHING_REDIS_SKIP_TLS_VERIFY = CACHING_REDIS.get('INSECURE_SKIP_TLS_VERIFY', Fa if LOGIN_TIMEOUT is not None: # Django default is 1209600 seconds (14 days) SESSION_COOKIE_AGE = LOGIN_TIMEOUT +SESSION_SAVE_EVERY_REQUEST = bool(LOGIN_PERSISTENCE) if SESSION_FILE_PATH is not None: SESSION_ENGINE = 'django.contrib.sessions.backends.file' From 1fc3c6d9d2a5cee27c6819bdc65d10b79e448493 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Fri, 20 Aug 2021 16:12:09 -0400 Subject: [PATCH 10/15] Fixes #6974: Show contextual label for IP address role --- docs/release-notes/version-2.11.md | 1 + netbox/templates/ipam/ipaddress.html | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 8db5ccd05..48e9f5257 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -13,6 +13,7 @@ * [#5968](https://github.com/netbox-community/netbox/issues/5968) - Model forms should save empty custom field values as null * [#6686](https://github.com/netbox-community/netbox/issues/6686) - Force assignment of null custom field values to objects +* [#6974](https://github.com/netbox-community/netbox/issues/6974) - Show contextual label for IP address role --- diff --git a/netbox/templates/ipam/ipaddress.html b/netbox/templates/ipam/ipaddress.html index f36d498d1..101f75dbd 100644 --- a/netbox/templates/ipam/ipaddress.html +++ b/netbox/templates/ipam/ipaddress.html @@ -56,7 +56,7 @@ Role {% if object.role %} - {{ object.get_role_display }} + {{ object.get_role_display }} {% else %} None {% endif %} From 8131feae8ac54c037e3a8c2b0534c37f6d10b80e Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 23 Aug 2021 09:36:05 -0400 Subject: [PATCH 11/15] Closes #7011: Add search field to VM interfaces filter form --- docs/release-notes/version-2.11.md | 1 + netbox/virtualization/forms.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 48e9f5257..a33a79e6d 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -8,6 +8,7 @@ * [#6790](https://github.com/netbox-community/netbox/issues/6790) - Recognize a /32 IPv4 address as a child of a /32 IPv4 prefix * [#6872](https://github.com/netbox-community/netbox/issues/6872) - Add table configuration button to child prefixes view * [#6929](https://github.com/netbox-community/netbox/issues/6929) - Introduce `LOGIN_PERSISTENCE` configuration parameter to persist user sessions +* [#7011](https://github.com/netbox-community/netbox/issues/7011) - Add search field to VM interfaces filter form ### Bug Fixes diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index 0819882c3..c40f2582c 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -834,6 +834,10 @@ class VMInterfaceBulkRenameForm(BulkRenameForm): class VMInterfaceFilterForm(BootstrapMixin, forms.Form): model = VMInterface + q = forms.CharField( + required=False, + label=_('Search') + ) cluster_id = DynamicModelMultipleChoiceField( queryset=Cluster.objects.all(), required=False, From cfa4f5677b040380c0477b532f5c4082b31c930f Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 23 Aug 2021 09:41:43 -0400 Subject: [PATCH 12/15] Fixes #7012: Fix hidden "add components" dropdown on devices list --- docs/release-notes/version-2.11.md | 1 + netbox/templates/dcim/device_list.html | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index a33a79e6d..5ca4be1dd 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -15,6 +15,7 @@ * [#5968](https://github.com/netbox-community/netbox/issues/5968) - Model forms should save empty custom field values as null * [#6686](https://github.com/netbox-community/netbox/issues/6686) - Force assignment of null custom field values to objects * [#6974](https://github.com/netbox-community/netbox/issues/6974) - Show contextual label for IP address role +* [#7012](https://github.com/netbox-community/netbox/issues/7012) - Fix hidden "add components" dropdown on devices list --- diff --git a/netbox/templates/dcim/device_list.html b/netbox/templates/dcim/device_list.html index 62d03493c..14b4d8b3b 100644 --- a/netbox/templates/dcim/device_list.html +++ b/netbox/templates/dcim/device_list.html @@ -2,7 +2,7 @@ {% block bulk_buttons %} {% if perms.dcim.change_device %} -
+
From 75c62ff7299ed789c35f0052b0c0712f29805976 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 23 Aug 2021 11:32:47 -0400 Subject: [PATCH 13/15] Print request index after webhook data dump --- netbox/extras/management/commands/webhook_receiver.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/netbox/extras/management/commands/webhook_receiver.py b/netbox/extras/management/commands/webhook_receiver.py index 147e4c261..382c830fd 100644 --- a/netbox/extras/management/commands/webhook_receiver.py +++ b/netbox/extras/management/commands/webhook_receiver.py @@ -37,12 +37,10 @@ class WebhookHandler(BaseHTTPRequestHandler): self.end_headers() self.wfile.write(b'Webhook received!\n') - request_counter += 1 - - # Print the request headers to stdout + # Print the request headers if self.show_headers: for k, v in self.headers.items(): - print('{}: {}'.format(k, v)) + print(f'{k}: {v}') print() # Print the request body (if any) @@ -55,8 +53,11 @@ class WebhookHandler(BaseHTTPRequestHandler): else: print('(No body)') + print(f'Completed request #{request_counter}') print('------------') + request_counter += 1 + class Command(BaseCommand): help = "Start a simple listener to display received HTTP requests" From 0b0ab9277cdb8f388f2d6aee92d708d5126b9111 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 23 Aug 2021 12:06:43 -0400 Subject: [PATCH 14/15] Fixes #6776: Fix erroneous webhook dispatch on failure to save objects --- docs/release-notes/version-2.11.md | 1 + netbox/extras/context_managers.py | 5 ++++- netbox/extras/signals.py | 16 ++++++++++++++++ netbox/netbox/views/generic.py | 15 ++++++++++++--- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index 5ca4be1dd..f1d709736 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -14,6 +14,7 @@ * [#5968](https://github.com/netbox-community/netbox/issues/5968) - Model forms should save empty custom field values as null * [#6686](https://github.com/netbox-community/netbox/issues/6686) - Force assignment of null custom field values to objects +* [#6776](https://github.com/netbox-community/netbox/issues/6776) - Fix erroneous webhook dispatch on failure to save objects * [#6974](https://github.com/netbox-community/netbox/issues/6974) - Show contextual label for IP address role * [#7012](https://github.com/netbox-community/netbox/issues/7012) - Fix hidden "add components" dropdown on devices list diff --git a/netbox/extras/context_managers.py b/netbox/extras/context_managers.py index 25a49b325..66b5ff94d 100644 --- a/netbox/extras/context_managers.py +++ b/netbox/extras/context_managers.py @@ -2,7 +2,7 @@ from contextlib import contextmanager from django.db.models.signals import m2m_changed, pre_delete, post_save -from extras.signals import _handle_changed_object, _handle_deleted_object +from extras.signals import clear_webhooks, _clear_webhook_queue, _handle_changed_object, _handle_deleted_object from utilities.utils import curry from .webhooks import flush_webhooks @@ -20,11 +20,13 @@ def change_logging(request): # Curry signals receivers to pass the current request handle_changed_object = curry(_handle_changed_object, request, webhook_queue) handle_deleted_object = curry(_handle_deleted_object, request, webhook_queue) + clear_webhook_queue = curry(_clear_webhook_queue, webhook_queue) # Connect our receivers to the post_save and post_delete signals. post_save.connect(handle_changed_object, dispatch_uid='handle_changed_object') m2m_changed.connect(handle_changed_object, dispatch_uid='handle_changed_object') pre_delete.connect(handle_deleted_object, dispatch_uid='handle_deleted_object') + clear_webhooks.connect(clear_webhook_queue, dispatch_uid='clear_webhook_queue') yield @@ -33,6 +35,7 @@ def change_logging(request): post_save.disconnect(handle_changed_object, dispatch_uid='handle_changed_object') m2m_changed.disconnect(handle_changed_object, dispatch_uid='handle_changed_object') pre_delete.disconnect(handle_deleted_object, dispatch_uid='handle_deleted_object') + clear_webhooks.disconnect(clear_webhook_queue, dispatch_uid='clear_webhook_queue') # Flush queued webhooks to RQ flush_webhooks(webhook_queue) diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py index e68b19a89..0dfef62f0 100644 --- a/netbox/extras/signals.py +++ b/netbox/extras/signals.py @@ -1,3 +1,4 @@ +import logging import random from datetime import timedelta @@ -6,6 +7,7 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.db import DEFAULT_DB_ALIAS from django.db.models.signals import m2m_changed, post_save, pre_delete +from django.dispatch import Signal from django.utils import timezone from django_prometheus.models import model_deletes, model_inserts, model_updates from prometheus_client import Counter @@ -19,6 +21,10 @@ from .webhooks import enqueue_object, get_snapshots, serialize_for_webhook # Change logging/webhooks # +# Define a custom signal that can be sent to clear any queued webhooks +clear_webhooks = Signal() + + def _handle_changed_object(request, webhook_queue, sender, instance, **kwargs): """ Fires when an object is created or updated. @@ -104,6 +110,16 @@ def _handle_deleted_object(request, webhook_queue, sender, instance, **kwargs): model_deletes.labels(instance._meta.model_name).inc() +def _clear_webhook_queue(webhook_queue, sender, **kwargs): + """ + Delete any queued webhooks (e.g. because of an aborted bulk transaction) + """ + logger = logging.getLogger('webhooks') + logger.info(f"Clearing {len(webhook_queue)} queued webhooks ({sender})") + + webhook_queue.clear() + + # # Custom fields # diff --git a/netbox/netbox/views/generic.py b/netbox/netbox/views/generic.py index ae2840a42..7570d26a3 100644 --- a/netbox/netbox/views/generic.py +++ b/netbox/netbox/views/generic.py @@ -17,6 +17,7 @@ from django.views.generic import View from django_tables2.export import TableExport from extras.models import CustomField, ExportTemplate +from extras.signals import clear_webhooks from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortTransaction, PermissionsViolation from utilities.forms import ( @@ -325,6 +326,7 @@ class ObjectEditView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): msg = "Object save failed due to object-level permissions violation" logger.debug(msg) form.add_error(None, msg) + clear_webhooks.send(sender=self) else: logger.debug("Form validation failed") @@ -603,12 +605,13 @@ class ObjectImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): raise ObjectDoesNotExist except AbortTransaction: - pass + clear_webhooks.send(sender=self) except PermissionsViolation: msg = "Object creation failed due to object-level permissions violation" logger.debug(msg) form.add_error(None, msg) + clear_webhooks.send(sender=self) if not model_form.errors: logger.info(f"Import object {obj} (PK: {obj.pk})") @@ -751,12 +754,13 @@ class BulkImportView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): }) except ValidationError: - pass + clear_webhooks.send(sender=self) except PermissionsViolation: msg = "Object import failed due to object-level permissions violation" logger.debug(msg) form.add_error(None, msg) + clear_webhooks.send(sender=self) else: logger.debug("Form validation failed") @@ -879,11 +883,13 @@ class BulkEditView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): except ValidationError as e: messages.error(self.request, "{} failed validation: {}".format(obj, e)) + clear_webhooks.send(sender=self) except PermissionsViolation: msg = "Object update failed due to object-level permissions violation" logger.debug(msg) form.add_error(None, msg) + clear_webhooks.send(sender=self) else: logger.debug("Form validation failed") @@ -987,6 +993,7 @@ class BulkRenameView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View): msg = "Object update failed due to object-level permissions violation" logger.debug(msg) form.add_error(None, msg) + clear_webhooks.send(sender=self) else: form = self.form(initial={'pk': request.POST.getlist('pk')}) @@ -1183,6 +1190,7 @@ class ComponentCreateView(GetReturnURLMixin, ObjectPermissionRequiredMixin, View msg = "Component creation failed due to object-level permissions violation" logger.debug(msg) form.add_error(None, msg) + clear_webhooks.send(sender=self) return render(request, self.template_name, { 'component_type': self.queryset.model._meta.verbose_name, @@ -1264,12 +1272,13 @@ class BulkComponentCreateView(GetReturnURLMixin, ObjectPermissionRequiredMixin, raise PermissionsViolation except IntegrityError: - pass + clear_webhooks.send(sender=self) except PermissionsViolation: msg = "Component creation failed due to object-level permissions violation" logger.debug(msg) form.add_error(None, msg) + clear_webhooks.send(sender=self) if not form.errors: msg = "Added {} {} to {} {}.".format( From 8497965cf7275807cccb031e46fcc5bf4ba82ab3 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Mon, 23 Aug 2021 12:49:32 -0400 Subject: [PATCH 15/15] Fixes #6326: Enable filtering assigned VLANs by group in interface edit form --- docs/release-notes/version-2.11.md | 1 + netbox/dcim/forms.py | 17 ++++++++++++++--- netbox/project-static/js/forms.js | 4 ++++ netbox/templates/dcim/interface_edit.html | 1 + .../virtualization/vminterface_edit.html | 1 + netbox/virtualization/forms.py | 17 ++++++++++++++--- 6 files changed, 35 insertions(+), 6 deletions(-) diff --git a/docs/release-notes/version-2.11.md b/docs/release-notes/version-2.11.md index f1d709736..e30a6827e 100644 --- a/docs/release-notes/version-2.11.md +++ b/docs/release-notes/version-2.11.md @@ -13,6 +13,7 @@ ### Bug Fixes * [#5968](https://github.com/netbox-community/netbox/issues/5968) - Model forms should save empty custom field values as null +* [#6326](https://github.com/netbox-community/netbox/issues/6326) - Enable filtering assigned VLANs by group in interface edit form * [#6686](https://github.com/netbox-community/netbox/issues/6686) - Force assignment of null custom field values to objects * [#6776](https://github.com/netbox-community/netbox/issues/6776) - Fix erroneous webhook dispatch on failure to save objects * [#6974](https://github.com/netbox-community/netbox/issues/6974) - Show contextual label for IP address role diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 81782f8ee..2eca6da08 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -18,7 +18,7 @@ from extras.forms import ( ) from extras.models import Tag from ipam.constants import BGP_ASN_MAX, BGP_ASN_MIN -from ipam.models import IPAddress, VLAN +from ipam.models import IPAddress, VLAN, VLANGroup from tenancy.forms import TenancyFilterForm, TenancyForm from tenancy.models import Tenant from utilities.forms import ( @@ -3109,15 +3109,26 @@ class InterfaceForm(BootstrapMixin, InterfaceCommonForm, CustomFieldModelForm): 'type': 'lag', } ) + vlan_group = DynamicModelChoiceField( + queryset=VLANGroup.objects.all(), + required=False, + label='VLAN group' + ) untagged_vlan = DynamicModelChoiceField( queryset=VLAN.objects.all(), required=False, - label='Untagged VLAN' + label='Untagged VLAN', + query_params={ + 'group_id': '$vlan_group', + } ) tagged_vlans = DynamicModelMultipleChoiceField( queryset=VLAN.objects.all(), required=False, - label='Tagged VLANs' + label='Tagged VLANs', + query_params={ + 'group_id': '$vlan_group', + } ) tags = DynamicModelMultipleChoiceField( queryset=Tag.objects.all(), diff --git a/netbox/project-static/js/forms.js b/netbox/project-static/js/forms.js index b95100acc..1045f1b4b 100644 --- a/netbox/project-static/js/forms.js +++ b/netbox/project-static/js/forms.js @@ -337,22 +337,26 @@ $(document).ready(function() { $('select#id_untagged_vlan').trigger('change'); $('select#id_tagged_vlans').val([]); $('select#id_tagged_vlans').trigger('change'); + $('select#id_vlan_group').parent().parent().hide(); $('select#id_untagged_vlan').parent().parent().hide(); $('select#id_tagged_vlans').parent().parent().hide(); } else if ($(this).val() == 'access') { $('select#id_tagged_vlans').val([]); $('select#id_tagged_vlans').trigger('change'); + $('select#id_vlan_group').parent().parent().show(); $('select#id_untagged_vlan').parent().parent().show(); $('select#id_tagged_vlans').parent().parent().hide(); } else if ($(this).val() == 'tagged') { + $('select#id_vlan_group').parent().parent().show(); $('select#id_untagged_vlan').parent().parent().show(); $('select#id_tagged_vlans').parent().parent().show(); } else if ($(this).val() == 'tagged-all') { $('select#id_tagged_vlans').val([]); $('select#id_tagged_vlans').trigger('change'); + $('select#id_vlan_group').parent().parent().show(); $('select#id_untagged_vlan').parent().parent().show(); $('select#id_tagged_vlans').parent().parent().hide(); } diff --git a/netbox/templates/dcim/interface_edit.html b/netbox/templates/dcim/interface_edit.html index 8a0c85a12..4dc081522 100644 --- a/netbox/templates/dcim/interface_edit.html +++ b/netbox/templates/dcim/interface_edit.html @@ -33,6 +33,7 @@
802.1Q Switching
{% render_field form.mode %} + {% render_field form.vlan_group %} {% render_field form.untagged_vlan %} {% render_field form.tagged_vlans %}
diff --git a/netbox/templates/virtualization/vminterface_edit.html b/netbox/templates/virtualization/vminterface_edit.html index f3ab4f9c2..5f0a56505 100644 --- a/netbox/templates/virtualization/vminterface_edit.html +++ b/netbox/templates/virtualization/vminterface_edit.html @@ -28,6 +28,7 @@
802.1Q Switching
{% render_field form.mode %} + {% render_field form.vlan_group %} {% render_field form.untagged_vlan %} {% render_field form.tagged_vlans %}
diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index c40f2582c..870de9b3c 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -12,7 +12,7 @@ from extras.forms import ( CustomFieldFilterForm, ) from extras.models import Tag -from ipam.models import IPAddress, VLAN +from ipam.models import IPAddress, VLAN, VLANGroup from tenancy.forms import TenancyFilterForm, TenancyForm from tenancy.models import Tenant from utilities.forms import ( @@ -616,15 +616,26 @@ class VMInterfaceForm(BootstrapMixin, InterfaceCommonForm, CustomFieldModelForm) required=False, label='Parent interface' ) + vlan_group = DynamicModelChoiceField( + queryset=VLANGroup.objects.all(), + required=False, + label='VLAN group' + ) untagged_vlan = DynamicModelChoiceField( queryset=VLAN.objects.all(), required=False, - label='Untagged VLAN' + label='Untagged VLAN', + query_params={ + 'group_id': '$vlan_group', + } ) tagged_vlans = DynamicModelMultipleChoiceField( queryset=VLAN.objects.all(), required=False, - label='Tagged VLANs' + label='Tagged VLANs', + query_params={ + 'group_id': '$vlan_group', + } ) tags = DynamicModelMultipleChoiceField( queryset=Tag.objects.all(),