From f1d5e28f13d3c8f7fe694860004b7b1eb169a717 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 10 Jan 2020 14:26:39 +0000 Subject: [PATCH 01/12] CSV import/export custom fields --- netbox/circuits/forms.py | 6 ++--- netbox/dcim/forms.py | 28 ++++++++++++------------ netbox/ipam/forms.py | 16 +++++++------- netbox/secrets/forms.py | 4 ++-- netbox/tenancy/forms.py | 4 ++-- netbox/utilities/templatetags/helpers.py | 3 +++ netbox/utilities/views.py | 17 ++++++++++++-- netbox/virtualization/forms.py | 8 +++---- 8 files changed, 51 insertions(+), 35 deletions(-) diff --git a/netbox/circuits/forms.py b/netbox/circuits/forms.py index 4a5c06a6e..c5decc9e7 100644 --- a/netbox/circuits/forms.py +++ b/netbox/circuits/forms.py @@ -46,7 +46,7 @@ class ProviderForm(BootstrapMixin, CustomFieldForm): } -class ProviderCSVForm(forms.ModelForm): +class ProviderCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -144,7 +144,7 @@ class CircuitTypeForm(BootstrapMixin, forms.ModelForm): ] -class CircuitTypeCSVForm(forms.ModelForm): +class CircuitTypeCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -187,7 +187,7 @@ class CircuitForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class CircuitCSVForm(forms.ModelForm): +class CircuitCSVForm(CustomFieldForm): provider = forms.ModelChoiceField( queryset=Provider.objects.all(), to_field_name='name', diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 6086491d0..5f10752bc 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -151,7 +151,7 @@ class RegionForm(BootstrapMixin, forms.ModelForm): } -class RegionCSVForm(forms.ModelForm): +class RegionCSVForm(CustomFieldForm): parent = forms.ModelChoiceField( queryset=Region.objects.all(), required=False, @@ -231,7 +231,7 @@ class SiteForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class SiteCSVForm(forms.ModelForm): +class SiteCSVForm(CustomFieldForm): status = CSVChoiceField( choices=SITE_STATUS_CHOICES, required=False, @@ -355,7 +355,7 @@ class RackGroupForm(BootstrapMixin, forms.ModelForm): } -class RackGroupCSVForm(forms.ModelForm): +class RackGroupCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', @@ -411,7 +411,7 @@ class RackRoleForm(BootstrapMixin, forms.ModelForm): ] -class RackRoleCSVForm(forms.ModelForm): +class RackRoleCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -472,7 +472,7 @@ class RackForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class RackCSVForm(forms.ModelForm): +class RackCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', @@ -852,7 +852,7 @@ class ManufacturerForm(BootstrapMixin, forms.ModelForm): ] -class ManufacturerCSVForm(forms.ModelForm): +class ManufacturerCSVForm(CustomFieldForm): class Meta: model = Manufacturer @@ -890,7 +890,7 @@ class DeviceTypeForm(BootstrapMixin, CustomFieldForm): } -class DeviceTypeCSVForm(forms.ModelForm): +class DeviceTypeCSVForm(CustomFieldForm): manufacturer = forms.ModelChoiceField( queryset=Manufacturer.objects.all(), required=True, @@ -1308,7 +1308,7 @@ class DeviceRoleForm(BootstrapMixin, forms.ModelForm): ] -class DeviceRoleCSVForm(forms.ModelForm): +class DeviceRoleCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -1342,7 +1342,7 @@ class PlatformForm(BootstrapMixin, forms.ModelForm): } -class PlatformCSVForm(forms.ModelForm): +class PlatformCSVForm(CustomFieldForm): slug = SlugField() manufacturer = forms.ModelChoiceField( queryset=Manufacturer.objects.all(), @@ -1564,7 +1564,7 @@ class DeviceForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.initial['rack'] = self.instance.parent_bay.device.rack_id -class BaseDeviceCSVForm(forms.ModelForm): +class BaseDeviceCSVForm(CustomFieldForm): device_role = forms.ModelChoiceField( queryset=DeviceRole.objects.all(), to_field_name='name', @@ -2919,7 +2919,7 @@ class CableForm(BootstrapMixin, forms.ModelForm): ] -class CableCSVForm(forms.ModelForm): +class CableCSVForm(CustomFieldForm): # Termination A side_a_device = FlexibleModelChoiceField( @@ -3294,7 +3294,7 @@ class InventoryItemForm(BootstrapMixin, forms.ModelForm): } -class InventoryItemCSVForm(forms.ModelForm): +class InventoryItemCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3623,7 +3623,7 @@ class PowerPanelForm(BootstrapMixin, forms.ModelForm): } -class PowerPanelCSVForm(forms.ModelForm): +class PowerPanelCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', @@ -3747,7 +3747,7 @@ class PowerFeedForm(BootstrapMixin, CustomFieldForm): self.initial['site'] = self.instance.power_panel.site -class PowerFeedCSVForm(forms.ModelForm): +class PowerFeedCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', diff --git a/netbox/ipam/forms.py b/netbox/ipam/forms.py index 413e72eaf..46346cf9c 100644 --- a/netbox/ipam/forms.py +++ b/netbox/ipam/forms.py @@ -48,7 +48,7 @@ class VRFForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class VRFCSVForm(forms.ModelForm): +class VRFCSVForm(CustomFieldForm): tenant = forms.ModelChoiceField( queryset=Tenant.objects.all(), required=False, @@ -118,7 +118,7 @@ class RIRForm(BootstrapMixin, forms.ModelForm): ] -class RIRCSVForm(forms.ModelForm): +class RIRCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -165,7 +165,7 @@ class AggregateForm(BootstrapMixin, CustomFieldForm): } -class AggregateCSVForm(forms.ModelForm): +class AggregateCSVForm(CustomFieldForm): rir = forms.ModelChoiceField( queryset=RIR.objects.all(), to_field_name='name', @@ -247,7 +247,7 @@ class RoleForm(BootstrapMixin, forms.ModelForm): ] -class RoleCSVForm(forms.ModelForm): +class RoleCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -340,7 +340,7 @@ class PrefixForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.fields['vrf'].empty_label = 'Global' -class PrefixCSVForm(forms.ModelForm): +class PrefixCSVForm(CustomFieldForm): vrf = FlexibleModelChoiceField( queryset=VRF.objects.all(), to_field_name='rd', @@ -759,7 +759,7 @@ class IPAddressBulkAddForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.fields['vrf'].empty_label = 'Global' -class IPAddressCSVForm(forms.ModelForm): +class IPAddressCSVForm(CustomFieldForm): vrf = FlexibleModelChoiceField( queryset=VRF.objects.all(), to_field_name='rd', @@ -1025,7 +1025,7 @@ class VLANGroupForm(BootstrapMixin, forms.ModelForm): } -class VLANGroupCSVForm(forms.ModelForm): +class VLANGroupCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), required=False, @@ -1122,7 +1122,7 @@ class VLANForm(BootstrapMixin, TenancyForm, CustomFieldForm): } -class VLANCSVForm(forms.ModelForm): +class VLANCSVForm(CustomFieldForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), required=False, diff --git a/netbox/secrets/forms.py b/netbox/secrets/forms.py index ed0f455c1..47fb27bbb 100644 --- a/netbox/secrets/forms.py +++ b/netbox/secrets/forms.py @@ -50,7 +50,7 @@ class SecretRoleForm(BootstrapMixin, forms.ModelForm): } -class SecretRoleCSVForm(forms.ModelForm): +class SecretRoleCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -113,7 +113,7 @@ class SecretForm(BootstrapMixin, CustomFieldForm): }) -class SecretCSVForm(forms.ModelForm): +class SecretCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', diff --git a/netbox/tenancy/forms.py b/netbox/tenancy/forms.py index f8aaa45e5..77f8305f3 100644 --- a/netbox/tenancy/forms.py +++ b/netbox/tenancy/forms.py @@ -23,7 +23,7 @@ class TenantGroupForm(BootstrapMixin, forms.ModelForm): ] -class TenantGroupCSVForm(forms.ModelForm): +class TenantGroupCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -57,7 +57,7 @@ class TenantForm(BootstrapMixin, CustomFieldForm): } -class TenantCSVForm(forms.ModelForm): +class TenantCSVForm(CustomFieldForm): slug = SlugField() group = forms.ModelChoiceField( queryset=TenantGroup.objects.all(), diff --git a/netbox/utilities/templatetags/helpers.py b/netbox/utilities/templatetags/helpers.py index 7b1e059a6..a13a5f2b0 100644 --- a/netbox/utilities/templatetags/helpers.py +++ b/netbox/utilities/templatetags/helpers.py @@ -149,6 +149,9 @@ def example_choices(field, arg=3): break if not value or not label: continue + # Handling for custom fields + if hasattr(label, 'value'): + label = label.value examples.append(label) return ', '.join(examples) or 'None' diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index 1aa358fba..96be35130 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -78,15 +78,28 @@ class ObjectListView(View): Export the queryset of objects as comma-separated value (CSV), using the model's to_csv() method. """ csv_data = [] + custom_fields = [] # Start with the column headers headers = ','.join(self.queryset.model.csv_headers) + + # Add custom field headers + content_type = ContentType.objects.get_for_model(self.queryset.model) + + for custom_field in CustomField.objects.filter(obj_type=content_type): + headers += ',cf_{}'.format(custom_field.name) + custom_fields.append(custom_field.name) + csv_data.append(headers) # Iterate through the queryset appending each object for obj in self.queryset: - data = csv_format(obj.to_csv()) - csv_data.append(data) + data = obj.to_csv() + + for custom_field in custom_fields: + data += (obj.cf.get(custom_field, ''),) + + csv_data.append(csv_format(data)) return csv_data diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index 427e676f6..90ba2e123 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -37,7 +37,7 @@ class ClusterTypeForm(BootstrapMixin, forms.ModelForm): ] -class ClusterTypeCSVForm(forms.ModelForm): +class ClusterTypeCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -62,7 +62,7 @@ class ClusterGroupForm(BootstrapMixin, forms.ModelForm): ] -class ClusterGroupCSVForm(forms.ModelForm): +class ClusterGroupCSVForm(CustomFieldForm): slug = SlugField() class Meta: @@ -101,7 +101,7 @@ class ClusterForm(BootstrapMixin, CustomFieldForm): } -class ClusterCSVForm(forms.ModelForm): +class ClusterCSVForm(CustomFieldForm): type = forms.ModelChoiceField( queryset=ClusterType.objects.all(), to_field_name='name', @@ -416,7 +416,7 @@ class VirtualMachineForm(BootstrapMixin, TenancyForm, CustomFieldForm): self.fields['primary_ip6'].widget.attrs['readonly'] = True -class VirtualMachineCSVForm(forms.ModelForm): +class VirtualMachineCSVForm(CustomFieldForm): status = CSVChoiceField( choices=VM_STATUS_CHOICES, required=False, From 37322fc1006e0f267c592408c2f2445068617cfb Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 10 Jan 2020 14:58:15 +0000 Subject: [PATCH 02/12] Fixed import choice name --- netbox/extras/forms.py | 6 +++--- netbox/utilities/forms.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 4f7f57fff..efb33905c 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -10,8 +10,8 @@ from dcim.models import DeviceRole, Platform, Region, Site from tenancy.models import Tenant, TenantGroup from utilities.forms import ( add_blank_choice, APISelectMultiple, BootstrapMixin, BulkEditForm, BulkEditNullBooleanSelect, ColorSelect, - CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, LaxURLField, JSONField, - SlugField, StaticSelect2, BOOLEAN_WITH_BLANK_CHOICES, + CSVCustomFieldChoiceField, CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, + LaxURLField, JSONField, SlugField, StaticSelect2, BOOLEAN_WITH_BLANK_CHOICES, ) from .constants import * from .models import ConfigContext, CustomField, CustomFieldValue, ImageAttachment, ObjectChange, Tag @@ -71,7 +71,7 @@ def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=F default_choice = cf.choices.get(value=initial).pk except ObjectDoesNotExist: pass - field = forms.TypedChoiceField( + field = CSVCustomFieldChoiceField( choices=choices, coerce=int, required=cf.required, initial=default_choice, widget=StaticSelect2() ) diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index eeee719ae..a98f29a8e 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -469,6 +469,23 @@ class CSVChoiceField(forms.ChoiceField): return self.choice_values[value] +class CSVCustomFieldChoiceField(forms.TypedChoiceField): + """ + Invert the provided set of choices to take the human-friendly label as input, and return the database value. + """ + + def __init__(self, choices, *args, **kwargs): + super().__init__(choices=choices, *args, **kwargs) + self.choice_values = {str(label): value for value, label in unpack_grouped_choices(choices)} + + def clean(self, value): + if not value: + return None + if value in self.choice_values: + return self.choice_values[value] + return super().clean(value) + + class ExpandableNameField(forms.CharField): """ A field which allows for numeric range expansion From de1355e6bc6c42f10dd135c89478322a924073a4 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 10 Jan 2020 15:00:57 +0000 Subject: [PATCH 03/12] Changelog #568 --- docs/release-notes/version-2.6.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release-notes/version-2.6.md b/docs/release-notes/version-2.6.md index 52b682746..6f4019460 100644 --- a/docs/release-notes/version-2.6.md +++ b/docs/release-notes/version-2.6.md @@ -2,6 +2,7 @@ ## Enhancements +* [#568](https://github.com/netbox-community/netbox/issues/568) - Allow custom fields to be imported and exported using CSV * [#1982](https://github.com/netbox-community/netbox/issues/1982) - Improved NAPALM method documentation in Swagger * [#2050](https://github.com/netbox-community/netbox/issues/2050) - Preview image attachments when hovering the link * [#2113](https://github.com/netbox-community/netbox/issues/2113) - Allow NAPALM driver settings to be changed with request headers From a2d5aca1d9a9a3ff717c0eacda279464a6a9d7ea Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 16 Jan 2020 16:05:45 +0000 Subject: [PATCH 04/12] Moved changelog to v2.7 --- docs/release-notes/version-2.6.md | 8 -------- docs/release-notes/version-2.7.md | 1 + 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/release-notes/version-2.6.md b/docs/release-notes/version-2.6.md index 51a397b3c..9fd258b0f 100644 --- a/docs/release-notes/version-2.6.md +++ b/docs/release-notes/version-2.6.md @@ -1,11 +1,3 @@ -# v2.6.13 (FUTURE) - -## Enhancements - -* [#568](https://github.com/netbox-community/netbox/issues/568) - Allow custom fields to be imported and exported using CSV - ---- - # v2.6.12 (2020-01-13) ## Enhancements diff --git a/docs/release-notes/version-2.7.md b/docs/release-notes/version-2.7.md index ac9d81e2c..800389228 100644 --- a/docs/release-notes/version-2.7.md +++ b/docs/release-notes/version-2.7.md @@ -221,6 +221,7 @@ PATCH) to maintain backward compatibility. This behavior will be discontinued be ## Enhancements * [#33](https://github.com/netbox-community/netbox/issues/33) - Add ability to clone objects (pre-populate form fields) +* [#568](https://github.com/netbox-community/netbox/issues/568) - Allow custom fields to be imported and exported using CSV * [#648](https://github.com/netbox-community/netbox/issues/648) - Pre-populate forms when selecting "create and add another" * [#792](https://github.com/netbox-community/netbox/issues/792) - Add power port and power outlet types * [#1865](https://github.com/netbox-community/netbox/issues/1865) - Add console port and console server port types From 9f68f8d1a615b210a8c33c76a68802a6c6bec1ea Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 16 Jan 2020 16:07:24 +0000 Subject: [PATCH 05/12] Update component CSV forms --- netbox/dcim/forms.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index a768039cf..9c3332ff3 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -2175,7 +2175,7 @@ class ConsolePortCreateForm(ComponentForm): ) -class ConsolePortCSVForm(forms.ModelForm): +class ConsolePortCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2267,7 +2267,7 @@ class ConsoleServerPortBulkDisconnectForm(ConfirmationForm): ) -class ConsoleServerPortCSVForm(forms.ModelForm): +class ConsoleServerPortCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2334,7 +2334,7 @@ class PowerPortCreateForm(ComponentForm): ) -class PowerPortCSVForm(forms.ModelForm): +class PowerPortCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2419,7 +2419,7 @@ class PowerOutletCreateForm(ComponentForm): self.fields['power_port'].queryset = PowerPort.objects.filter(device=self.parent) -class PowerOutletCSVForm(forms.ModelForm): +class PowerOutletCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2668,7 +2668,7 @@ class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): self.fields['lag'].queryset = Interface.objects.none() -class InterfaceCSVForm(forms.ModelForm): +class InterfaceCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), required=False, @@ -2925,7 +2925,7 @@ class FrontPortCreateForm(ComponentForm): } -class FrontPortCSVForm(forms.ModelForm): +class FrontPortCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3051,7 +3051,7 @@ class RearPortCreateForm(ComponentForm): ) -class RearPortCSVForm(forms.ModelForm): +class RearPortCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3655,7 +3655,7 @@ class PopulateDeviceBayForm(BootstrapMixin, forms.Form): ).exclude(pk=device_bay.device.pk) -class DeviceBayCSVForm(forms.ModelForm): +class DeviceBayCSVForm(CustomFieldForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', From 9128435113e35bc6941a71fd9c4ee93d8e973f87 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 23 Jan 2020 17:03:14 +0000 Subject: [PATCH 06/12] Removed CustomFieldForm class from models without custom fields --- netbox/circuits/forms.py | 2 +- netbox/dcim/forms.py | 34 +++++++++++++++++----------------- netbox/ipam/forms.py | 6 +++--- netbox/secrets/forms.py | 2 +- netbox/tenancy/forms.py | 2 +- netbox/virtualization/forms.py | 4 ++-- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/netbox/circuits/forms.py b/netbox/circuits/forms.py index 5416a055f..d14403815 100644 --- a/netbox/circuits/forms.py +++ b/netbox/circuits/forms.py @@ -144,7 +144,7 @@ class CircuitTypeForm(BootstrapMixin, forms.ModelForm): ] -class CircuitTypeCSVForm(CustomFieldForm): +class CircuitTypeCSVForm(forms.ModelForm): slug = SlugField() class Meta: diff --git a/netbox/dcim/forms.py b/netbox/dcim/forms.py index 75a0992ec..a5e8a782f 100644 --- a/netbox/dcim/forms.py +++ b/netbox/dcim/forms.py @@ -184,7 +184,7 @@ class RegionForm(BootstrapMixin, forms.ModelForm): } -class RegionCSVForm(CustomFieldForm): +class RegionCSVForm(forms.ModelForm): parent = forms.ModelChoiceField( queryset=Region.objects.all(), required=False, @@ -388,7 +388,7 @@ class RackGroupForm(BootstrapMixin, forms.ModelForm): } -class RackGroupCSVForm(CustomFieldForm): +class RackGroupCSVForm(forms.ModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', @@ -444,7 +444,7 @@ class RackRoleForm(BootstrapMixin, forms.ModelForm): ] -class RackRoleCSVForm(CustomFieldForm): +class RackRoleCSVForm(forms.ModelForm): slug = SlugField() class Meta: @@ -882,7 +882,7 @@ class ManufacturerForm(BootstrapMixin, forms.ModelForm): ] -class ManufacturerCSVForm(CustomFieldForm): +class ManufacturerCSVForm(forms.ModelForm): class Meta: model = Manufacturer @@ -1458,7 +1458,7 @@ class DeviceRoleForm(BootstrapMixin, forms.ModelForm): ] -class DeviceRoleCSVForm(CustomFieldForm): +class DeviceRoleCSVForm(forms.ModelForm): slug = SlugField() class Meta: @@ -1492,7 +1492,7 @@ class PlatformForm(BootstrapMixin, forms.ModelForm): } -class PlatformCSVForm(CustomFieldForm): +class PlatformCSVForm(forms.ModelForm): slug = SlugField() manufacturer = forms.ModelChoiceField( queryset=Manufacturer.objects.all(), @@ -2179,7 +2179,7 @@ class ConsolePortCreateForm(ComponentForm): ) -class ConsolePortCSVForm(CustomFieldForm): +class ConsolePortCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2271,7 +2271,7 @@ class ConsoleServerPortBulkDisconnectForm(ConfirmationForm): ) -class ConsoleServerPortCSVForm(CustomFieldForm): +class ConsoleServerPortCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2338,7 +2338,7 @@ class PowerPortCreateForm(ComponentForm): ) -class PowerPortCSVForm(CustomFieldForm): +class PowerPortCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2423,7 +2423,7 @@ class PowerOutletCreateForm(ComponentForm): self.fields['power_port'].queryset = PowerPort.objects.filter(device=self.parent) -class PowerOutletCSVForm(CustomFieldForm): +class PowerOutletCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -2672,7 +2672,7 @@ class InterfaceCreateForm(InterfaceCommonForm, ComponentForm, forms.Form): self.fields['lag'].queryset = Interface.objects.none() -class InterfaceCSVForm(CustomFieldForm): +class InterfaceCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), required=False, @@ -2929,7 +2929,7 @@ class FrontPortCreateForm(ComponentForm): } -class FrontPortCSVForm(CustomFieldForm): +class FrontPortCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3055,7 +3055,7 @@ class RearPortCreateForm(ComponentForm): ) -class RearPortCSVForm(CustomFieldForm): +class RearPortCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3365,7 +3365,7 @@ class CableForm(BootstrapMixin, forms.ModelForm): ] -class CableCSVForm(CustomFieldForm): +class CableCSVForm(forms.ModelForm): # Termination A side_a_device = FlexibleModelChoiceField( @@ -3659,7 +3659,7 @@ class PopulateDeviceBayForm(BootstrapMixin, forms.Form): ).exclude(pk=device_bay.device.pk) -class DeviceBayCSVForm(CustomFieldForm): +class DeviceBayCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -3801,7 +3801,7 @@ class InventoryItemForm(BootstrapMixin, forms.ModelForm): } -class InventoryItemCSVForm(CustomFieldForm): +class InventoryItemCSVForm(forms.ModelForm): device = FlexibleModelChoiceField( queryset=Device.objects.all(), to_field_name='name', @@ -4130,7 +4130,7 @@ class PowerPanelForm(BootstrapMixin, forms.ModelForm): } -class PowerPanelCSVForm(CustomFieldForm): +class PowerPanelCSVForm(forms.ModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), to_field_name='name', diff --git a/netbox/ipam/forms.py b/netbox/ipam/forms.py index d2efc2bcc..3bafad51b 100644 --- a/netbox/ipam/forms.py +++ b/netbox/ipam/forms.py @@ -118,7 +118,7 @@ class RIRForm(BootstrapMixin, forms.ModelForm): ] -class RIRCSVForm(CustomFieldForm): +class RIRCSVForm(forms.ModelForm): slug = SlugField() class Meta: @@ -247,7 +247,7 @@ class RoleForm(BootstrapMixin, forms.ModelForm): ] -class RoleCSVForm(CustomFieldForm): +class RoleCSVForm(forms.ModelForm): slug = SlugField() class Meta: @@ -1026,7 +1026,7 @@ class VLANGroupForm(BootstrapMixin, forms.ModelForm): } -class VLANGroupCSVForm(CustomFieldForm): +class VLANGroupCSVForm(forms.ModelForm): site = forms.ModelChoiceField( queryset=Site.objects.all(), required=False, diff --git a/netbox/secrets/forms.py b/netbox/secrets/forms.py index 0b2cb3cb4..bddb1c109 100644 --- a/netbox/secrets/forms.py +++ b/netbox/secrets/forms.py @@ -52,7 +52,7 @@ class SecretRoleForm(BootstrapMixin, forms.ModelForm): } -class SecretRoleCSVForm(CustomFieldForm): +class SecretRoleCSVForm(forms.ModelForm): slug = SlugField() class Meta: diff --git a/netbox/tenancy/forms.py b/netbox/tenancy/forms.py index 77f8305f3..22deae434 100644 --- a/netbox/tenancy/forms.py +++ b/netbox/tenancy/forms.py @@ -23,7 +23,7 @@ class TenantGroupForm(BootstrapMixin, forms.ModelForm): ] -class TenantGroupCSVForm(CustomFieldForm): +class TenantGroupCSVForm(forms.ModelForm): slug = SlugField() class Meta: diff --git a/netbox/virtualization/forms.py b/netbox/virtualization/forms.py index 457369851..27c22fa71 100644 --- a/netbox/virtualization/forms.py +++ b/netbox/virtualization/forms.py @@ -33,7 +33,7 @@ class ClusterTypeForm(BootstrapMixin, forms.ModelForm): ] -class ClusterTypeCSVForm(CustomFieldForm): +class ClusterTypeCSVForm(forms.ModelForm): slug = SlugField() class Meta: @@ -58,7 +58,7 @@ class ClusterGroupForm(BootstrapMixin, forms.ModelForm): ] -class ClusterGroupCSVForm(CustomFieldForm): +class ClusterGroupCSVForm(forms.ModelForm): slug = SlugField() class Meta: From 0ab19d723dc75975c1c5840d9d91abfbbaece950 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 23 Jan 2020 17:18:58 +0000 Subject: [PATCH 07/12] Moved the header join logic after the custom fields are added --- netbox/utilities/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index 67af4371f..4801d61a3 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -91,16 +91,16 @@ class ObjectListView(View): custom_fields = [] # Start with the column headers - headers = ','.join(self.queryset.model.csv_headers) + headers = self.queryset.model.csv_headers.copy() # Add custom field headers content_type = ContentType.objects.get_for_model(self.queryset.model) for custom_field in CustomField.objects.filter(obj_type=content_type): - headers += ',cf_{}'.format(custom_field.name) + headers.append(custom_field.name) custom_fields.append(custom_field.name) - csv_data.append(headers) + csv_data.append(','.join(headers)) # Iterate through the queryset appending each object for obj in self.queryset: From 0a5eecd0e3114043f4939a4e473c65eaf762b88a Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 23 Jan 2020 17:37:51 +0000 Subject: [PATCH 08/12] Explicitly use the value of the choice, instead of relying on __str__ --- netbox/extras/forms.py | 2 +- netbox/utilities/templatetags/helpers.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 17ae6c907..6aa0e3552 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -61,7 +61,7 @@ def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=F # Select elif cf.type == CustomFieldTypeChoices.TYPE_SELECT: - choices = [(cfc.pk, cfc) for cfc in cf.choices.all()] + choices = [(cfc.pk, cfc.value) for cfc in cf.choices.all()] if not cf.required or bulk_edit or filterable_only: choices = [(None, '---------')] + choices # Check for a default choice diff --git a/netbox/utilities/templatetags/helpers.py b/netbox/utilities/templatetags/helpers.py index 19a26b3c9..c4b3bb6ea 100644 --- a/netbox/utilities/templatetags/helpers.py +++ b/netbox/utilities/templatetags/helpers.py @@ -149,9 +149,6 @@ def example_choices(field, arg=3): break if not value or not label: continue - # Handling for custom fields - if hasattr(label, 'value'): - label = label.value examples.append(label) return ', '.join(examples) or 'None' From 8f86244b4fe0c38134022a90f80151f6e446fcd9 Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 23 Jan 2020 18:54:37 +0000 Subject: [PATCH 09/12] Cleaned the CustomField choice field --- netbox/extras/forms.py | 4 ++-- netbox/utilities/forms.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/netbox/extras/forms.py b/netbox/extras/forms.py index 6aa0e3552..314455b83 100644 --- a/netbox/extras/forms.py +++ b/netbox/extras/forms.py @@ -10,7 +10,7 @@ from dcim.models import DeviceRole, Platform, Region, Site from tenancy.models import Tenant, TenantGroup from utilities.forms import ( add_blank_choice, APISelectMultiple, BootstrapMixin, BulkEditForm, BulkEditNullBooleanSelect, ColorSelect, - CSVCustomFieldChoiceField, CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, + CustomFieldChoiceField, CommentField, ContentTypeSelect, DatePicker, DateTimePicker, FilterChoiceField, LaxURLField, JSONField, SlugField, StaticSelect2, BOOLEAN_WITH_BLANK_CHOICES, ) from .choices import * @@ -71,7 +71,7 @@ def get_custom_fields_for_model(content_type, filterable_only=False, bulk_edit=F default_choice = cf.choices.get(value=initial).pk except ObjectDoesNotExist: pass - field = CSVCustomFieldChoiceField( + field = CustomFieldChoiceField( choices=choices, coerce=int, required=cf.required, initial=default_choice, widget=StaticSelect2() ) diff --git a/netbox/utilities/forms.py b/netbox/utilities/forms.py index 453561303..1dd2c06a7 100644 --- a/netbox/utilities/forms.py +++ b/netbox/utilities/forms.py @@ -442,18 +442,18 @@ class CSVChoiceField(forms.ChoiceField): return self.choice_values[value] -class CSVCustomFieldChoiceField(forms.TypedChoiceField): +class CustomFieldChoiceField(forms.TypedChoiceField): """ - Invert the provided set of choices to take the human-friendly label as input, and return the database value. + Accept human-friendly label as input, and return the database value. If the label is not matched, the normal, + value-based input is assumed. """ def __init__(self, choices, *args, **kwargs): super().__init__(choices=choices, *args, **kwargs) - self.choice_values = {str(label): value for value, label in unpack_grouped_choices(choices)} + self.choice_values = {label: value for value, label in unpack_grouped_choices(choices)} def clean(self, value): - if not value: - return None + # Check if the value is actually a label if value in self.choice_values: return self.choice_values[value] return super().clean(value) From bed08a7b07eccbb6797cfe7ee29dbb38a1a1976e Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Thu, 23 Jan 2020 20:26:21 +0000 Subject: [PATCH 10/12] Use model's `get_custom_fields` --- netbox/utilities/views.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/netbox/utilities/views.py b/netbox/utilities/views.py index 4801d61a3..d900a8545 100644 --- a/netbox/utilities/views.py +++ b/netbox/utilities/views.py @@ -93,12 +93,11 @@ class ObjectListView(View): # Start with the column headers headers = self.queryset.model.csv_headers.copy() - # Add custom field headers - content_type = ContentType.objects.get_for_model(self.queryset.model) - - for custom_field in CustomField.objects.filter(obj_type=content_type): - headers.append(custom_field.name) - custom_fields.append(custom_field.name) + # Add custom field headers, if any + if hasattr(self.queryset.model, 'get_custom_fields'): + for custom_field in self.queryset.model().get_custom_fields(): + headers.append(custom_field.name) + custom_fields.append(custom_field.name) csv_data.append(','.join(headers)) From c22024b618b32928a98ee18398f10c7f9bfb27ca Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 24 Jan 2020 22:15:09 +0000 Subject: [PATCH 11/12] Added CSV import test --- netbox/extras/tests/test_customfields.py | 64 +++++++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 192958840..7871e031e 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -1,14 +1,14 @@ from datetime import date from django.contrib.contenttypes.models import ContentType -from django.test import TestCase +from django.test import Client, TestCase from django.urls import reverse from rest_framework import status from dcim.models import Site from extras.choices import * from extras.models import CustomField, CustomFieldValue, CustomFieldChoice -from utilities.testing import APITestCase +from utilities.testing import APITestCase, create_test_user from virtualization.models import VirtualMachine @@ -364,3 +364,63 @@ class CustomFieldChoiceAPITest(APITestCase): self.assertEqual(self.cf_choice_1.pk, response.data[self.cf_1.name][self.cf_choice_1.value]) self.assertEqual(self.cf_choice_2.pk, response.data[self.cf_1.name][self.cf_choice_2.value]) self.assertEqual(self.cf_choice_3.pk, response.data[self.cf_2.name][self.cf_choice_3.value]) + + +class CustomFieldCSV(TestCase): + def setUp(self): + super().setUp() + + user = create_test_user( + permissions=[ + 'dcim.view_site', + 'dcim.add_site', + ] + ) + self.client = Client() + self.client.force_login(user) + + obj_type = ContentType.objects.get_for_model(Site) + + self.cf_text = CustomField.objects.create(name="text", type=CustomFieldTypeChoices.TYPE_TEXT) + self.cf_text.obj_type.set([obj_type]) + self.cf_text.save() + + self.cf_choice = CustomField.objects.create(name="choice", type=CustomFieldTypeChoices.TYPE_SELECT) + self.cf_choice.obj_type.set([obj_type]) + self.cf_choice.save() + + self.cf_choice_1 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_1") + self.cf_choice_2 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_2") + self.cf_choice_3 = CustomFieldChoice.objects.create(field=self.cf_choice, value="cf_field_3") + + def test_import(self): + """ + Import a site with custom fields + """ + csv_data = ( + "name,slug,cf_text,cf_choice", + "Site 1,site-1,something,cf_field_1", + ) + + response = self.client.post(reverse('dcim:site_import'), {'csv': '\n'.join(csv_data)}) + self.assertEqual(response.status_code, 200) + + site1_custom_fields = Site.objects.get(name='Site 1').get_custom_fields() + self.assertEqual(len(site1_custom_fields), 2) + self.assertEqual(site1_custom_fields[self.cf_text], 'something') + self.assertEqual(site1_custom_fields[self.cf_choice], self.cf_choice_1) + + + def test_import_invalid_choice(self): + """ + Import a site with an invalid choice + """ + csv_data = ( + "name,slug,cf_choice", + "Site 2,site-2,cf_field_4", + ) + + response = self.client.post(reverse('dcim:site_import'), {'csv': '\n'.join(csv_data)}) + self.assertEqual(response.status_code, 200) + + self.assertFalse(len(Site.objects.filter(name="Site 2")), 0) From 8ec0ad96bddbc98c724ce2072a96f0e5844cf30a Mon Sep 17 00:00:00 2001 From: Saria Hajjar Date: Fri, 24 Jan 2020 22:20:41 +0000 Subject: [PATCH 12/12] Formatting --- netbox/extras/tests/test_customfields.py | 1 - 1 file changed, 1 deletion(-) diff --git a/netbox/extras/tests/test_customfields.py b/netbox/extras/tests/test_customfields.py index 7871e031e..005f049b5 100644 --- a/netbox/extras/tests/test_customfields.py +++ b/netbox/extras/tests/test_customfields.py @@ -410,7 +410,6 @@ class CustomFieldCSV(TestCase): self.assertEqual(site1_custom_fields[self.cf_text], 'something') self.assertEqual(site1_custom_fields[self.cf_choice], self.cf_choice_1) - def test_import_invalid_choice(self): """ Import a site with an invalid choice