4347 Add JSON/YAML import support for all objects (#10367)

* 4347 initial code for json import

* 4347 initial code for json import

* Clean up form processing logic

* Consolidate import forms

* Consolidate object import/update logic

* Clean up bulk import view

Co-authored-by: jeremystretch <jstretch@ns1.com>

netbox/utilities/forms/choices.py

@@ -0,0 +1,17 @@
+from utilities.choices import ChoiceSet
+
+
+#
+# Import Choices
+#
+
+class ImportFormatChoices(ChoiceSet):
+    CSV = 'csv'
+    JSON = 'json'
+    YAML = 'yaml'
+
+    CHOICES = [
+        (CSV, 'CSV'),
+        (JSON, 'JSON'),
+        (YAML, 'YAML'),
+    ]

netbox/utilities/forms/forms.py

@@ -1,12 +1,15 @@
+import csv
 import json
 import re
+from io import StringIO
 
 import yaml
 from django import forms
+from utilities.forms.utils import parse_csv
 
+from .choices import ImportFormatChoices
 from .widgets import APISelect, APISelectMultiple, ClearableFileInput, StaticSelect
 
-
 __all__ = (
     'BootstrapMixin',
     'BulkEditForm',
@@ -120,64 +123,94 @@ class CSVModelForm(forms.ModelForm):
     """
     ModelForm used for the import of objects in CSV format.
     """
-
-    def __init__(self, *args, headers=None, **kwargs):
+    def __init__(self, *args, headers=None, fields=None, **kwargs):
+        headers = headers or {}
+        fields = fields or []
         super().__init__(*args, **kwargs)
 
         # Modify the model form to accommodate any customized to_field_name properties
-        if headers:
-            for field, to_field in headers.items():
-                if to_field is not None:
-                    self.fields[field].to_field_name = to_field
+        for field, to_field in headers.items():
+            if to_field is not None:
+                self.fields[field].to_field_name = to_field
+
+        # Omit any fields not specified (e.g. because the form is being used to
+        # updated rather than create objects)
+        if fields:
+            for field in list(self.fields.keys()):
+                if field not in fields:
+                    del self.fields[field]
 
 
 class ImportForm(BootstrapMixin, forms.Form):
-    """
-    Generic form for creating an object from JSON/YAML data
-    """
     data = forms.CharField(
+        required=False,
         widget=forms.Textarea(attrs={'class': 'font-monospace'}),
-        help_text="Enter object data in JSON or YAML format. Note: Only a single object/document is supported."
+        help_text="Enter object data in CSV, JSON or YAML format."
     )
+    data_file = forms.FileField(
+        label="Data file",
+        required=False
+    )
+    # TODO: Enable auto-detection of format
     format = forms.ChoiceField(
-        choices=(
-            ('json', 'JSON'),
-            ('yaml', 'YAML')
-        ),
-        initial='yaml'
+        choices=ImportFormatChoices,
+        initial=ImportFormatChoices.CSV,
+        widget=StaticSelect()
     )
 
+    data_field = 'data'
+
     def clean(self):
         super().clean()
-
-        data = self.cleaned_data['data']
         format = self.cleaned_data['format']
 
-        # Process JSON/YAML data
-        if format == 'json':
-            try:
-                self.cleaned_data['data'] = json.loads(data)
-                # Check for multiple JSON objects
-                if type(self.cleaned_data['data']) is not dict:
-                    raise forms.ValidationError({
-                        'data': "Import is limited to one object at a time."
-                    })
-            except json.decoder.JSONDecodeError as err:
-                raise forms.ValidationError({
-                    'data': "Invalid JSON data: {}".format(err)
-                })
+        # Determine whether we're reading from form data or an uploaded file
+        if self.cleaned_data['data'] and self.cleaned_data['data_file']:
+            raise forms.ValidationError("Form data must be empty when uploading a file.")
+        if 'data_file' in self.files:
+            self.data_field = 'data_file'
+            file = self.files.get('data_file')
+            data = file.read().decode('utf-8')
         else:
-            # Check for multiple YAML documents
-            if '\n---' in data:
-                raise forms.ValidationError({
-                    'data': "Import is limited to one object at a time."
-                })
-            try:
-                self.cleaned_data['data'] = yaml.load(data, Loader=yaml.SafeLoader)
-            except yaml.error.YAMLError as err:
-                raise forms.ValidationError({
-                    'data': "Invalid YAML data: {}".format(err)
-                })
+            data = self.cleaned_data['data']
+
+        # Process data according to the selected format
+        if format == ImportFormatChoices.CSV:
+            self.cleaned_data['data'] = self._clean_csv(data)
+        elif format == ImportFormatChoices.JSON:
+            self.cleaned_data['data'] = self._clean_json(data)
+        elif format == ImportFormatChoices.YAML:
+            self.cleaned_data['data'] = self._clean_yaml(data)
+
+    def _clean_csv(self, data):
+        stream = StringIO(data.strip())
+        reader = csv.reader(stream)
+        headers, records = parse_csv(reader)
+
+        # Set CSV headers for reference by the model form
+        self._csv_headers = headers
+
+        return records
+
+    def _clean_json(self, data):
+        try:
+            data = json.loads(data)
+            # Accommodate for users entering single objects
+            if type(data) is not list:
+                data = [data]
+            return data
+        except json.decoder.JSONDecodeError as err:
+            raise forms.ValidationError({
+                self.data_field: f"Invalid JSON data: {err}"
+            })
+
+    def _clean_yaml(self, data):
+        try:
+            return yaml.load_all(data, Loader=yaml.SafeLoader)
+        except yaml.error.YAMLError as err:
+            raise forms.ValidationError({
+                self.data_field: f"Invalid YAML data: {err}"
+            })
 
 
 class FilterForm(BootstrapMixin, forms.Form):