mirror/netbox-community-netbox
1
0
Fork 0
mirror of https://github.com/netbox-community/netbox.git synced 2024-05-10 07:54:54 +00:00
Code Activity

Closes #11780: Enable loading import data from remote sources

Browse Source
This commit is contained in:
jeremystretch
2023-03-14 14:53:26 -04:00
parent 8bd0a2ef9d
commit 1446b07f8c
7 changed files with 221 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/release-notes/version-3.5.md
View File

@ -40,6 +40,7 @@ A new ASN range model has been introduced to facilitate the provisioning of new
* [#11584](https://github.com/netbox-community/netbox/issues/11584) - Add a list view for contact assignments
* [#11625](https://github.com/netbox-community/netbox/issues/11625) - Add HTMX support to ObjectEditView
* [#11693](https://github.com/netbox-community/netbox/issues/11693) - Enable syncing export template content from remote sources
* [#11780](https://github.com/netbox-community/netbox/issues/11780) - Enable loading import data from remote sources
* [#11968](https://github.com/netbox-community/netbox/issues/11968) - Add navigation menu buttons to create device & VM components
### Other Changes

4
netbox/netbox/views/generic/bulk_views.py
View File

@ -16,10 +16,10 @@ from django_tables2.export import TableExport
from extras.models import ExportTemplate
from extras.signals import clear_webhooks
from utilities.choices import ImportFormatChoices
from utilities.error_handlers import handle_protectederror
from utilities.exceptions import AbortRequest, AbortTransaction, PermissionsViolation
from utilities.forms import BulkRenameForm, ConfirmationForm, ImportForm, restrict_form_fields
from utilities.forms import BulkRenameForm, ConfirmationForm, restrict_form_fields
from utilities.forms.bulk_import import ImportForm
from utilities.htmx import is_embedded, is_htmx
from utilities.permissions import get_permission_for_model
from utilities.views import GetReturnURLMixin

44
netbox/templates/generic/bulk_import.html
View File

@ -15,15 +15,20 @@ Context:
{% block tabs %}
<ul class="nav nav-tabs px-3">
<li class="nav-item" role="presentation">
<button class="nav-link active" id="data-import-tab" data-bs-toggle="tab" data-bs-target="#data-import-form" type="button" role="tab" aria-controls="data-import-form" aria-selected="true">
Data Import
<button class="nav-link active" id="import-form-tab" data-bs-toggle="tab" data-bs-target="#import-form" type="button" role="tab" aria-controls="import-form" aria-selected="true">
Direct Import
</button>
</li>
<li class="nav-item" role="presentation">
<button class="nav-link" id="file-upload-tab" data-bs-toggle="tab" data-bs-target="#file-upload-form" type="button" role="tab" aria-controls="file-upload-form" aria-selected="false">
<button class="nav-link" id="upload-form-tab" data-bs-toggle="tab" data-bs-target="#upload-form" type="button" role="tab" aria-controls="upload-form" aria-selected="false">
Upload File
</button>
</li>
<li class="nav-item" role="presentation">
<button class="nav-link" id="datafile-form-tab" data-bs-toggle="tab" data-bs-target="#datafile-form" type="button" role="tab" aria-controls="datafile-form" aria-selected="false">
Data File
</button>
</li>
</ul>
{% endblock tabs %}
@ -31,12 +36,12 @@ Context:
<div class="tab-content">
{# Data Import Form #}
<div class="tab-pane show active" id="data-import-form" role="tabpanel" aria-labelledby="data-import-tab">
{% block content %}
<div class="tab-pane show active" id="import-form" role="tabpanel" aria-labelledby="import-form-tab">
<div class="row">
<div class="col col-md-12 col-lg-10">
<div class="col col-md-12 col-lg-10 offset-lg-1">
<form action="" method="post" enctype="multipart/form-data" class="form">
{% csrf_token %}
<input type="hidden" name="import_method" value="direct" />
{% render_field form.data %}
{% render_field form.format %}
<div class="form-group">
@ -50,14 +55,35 @@ Context:
</form>
</div>
</div>
{% endblock content %}
</div>
{# File Upload Form #}
<div class="tab-pane show" id="file-upload-form" role="tabpanel" aria-labelledby="file-upload-tab">
<div class="col col-md-12 col-lg-10">
<div class="tab-pane show" id="upload-form" role="tabpanel" aria-labelledby="upload-form-tab">
<div class="col col-md-12 col-lg-10 offset-lg-1">
<form action="" method="post" enctype="multipart/form-data" class="form">
{% csrf_token %}
<input type="hidden" name="import_method" value="upload" />
{% render_field form.upload_file %}
{% render_field form.format %}
<div class="form-group">
<div class="col col-md-12 text-end">
<button type="submit" name="file_submit" class="btn btn-primary">Submit</button>
{% if return_url %}
<a href="{{ return_url }}" class="btn btn-outline-danger">Cancel</a>
{% endif %}
</div>
</div>
</form>
</div>
</div>
{# DataFile Form #}
<div class="tab-pane show" id="datafile-form" role="tabpanel" aria-labelledby="datafile-form-tab">
<div class="col col-md-12 col-lg-10 offset-lg-1">
<form action="" method="post" enctype="multipart/form-data" class="form">
{% csrf_token %}
<input type="hidden" name="import_method" value="datafile" />
{% render_field form.data_source %}
{% render_field form.data_file %}
{% render_field form.format %}
<div class="form-group">

12
netbox/utilities/choices.py
View File

@ -203,6 +203,18 @@ class ButtonColorChoices(ChoiceSet):
# Import Choices
#
class ImportMethodChoices(ChoiceSet):
DIRECT = 'direct'
UPLOAD = 'upload'
DATA_FILE = 'datafile'
CHOICES = [
(DIRECT, 'Direct'),
(UPLOAD, 'Upload'),
(DATA_FILE, 'Data file'),
]
class ImportFormatChoices(ChoiceSet):
AUTO = 'auto'
CSV = 'csv'

141
netbox/utilities/forms/bulk_import.py Normal file
View File

@ -0,0 +1,141 @@
import csv
import json
from io import StringIO
import yaml
from django import forms
from django.utils.translation import gettext as _
from extras.forms.mixins import SyncedDataMixin
from utilities.choices import ImportFormatChoices
from utilities.forms.utils import parse_csv
from ..choices import ImportMethodChoices
from .forms import BootstrapMixin
class ImportForm(BootstrapMixin, SyncedDataMixin, forms.Form):
import_method = forms.ChoiceField(
choices=ImportMethodChoices
)
data = forms.CharField(
required=False,
widget=forms.Textarea(attrs={'class': 'font-monospace'}),
help_text=_("Enter object data in CSV, JSON or YAML format.")
)
upload_file = forms.FileField(
label="Data file",
required=False
)
format = forms.ChoiceField(
choices=ImportFormatChoices,
initial=ImportFormatChoices.AUTO
)
data_field = 'data'
def clean(self):
super().clean()
# Determine import method
import_method = self.cleaned_data['import_method']
# Determine whether we're reading from form data or an uploaded file
if self.cleaned_data['data'] and import_method != ImportMethodChoices.DIRECT:
raise forms.ValidationError("Form data must be empty when uploading/selecting a file.")
if import_method == ImportMethodChoices.UPLOAD:
self.upload_file = 'upload_file'
file = self.files.get('upload_file')
data = file.read().decode('utf-8-sig')
elif import_method == ImportMethodChoices.DATA_FILE:
data = self.cleaned_data['data_file'].data_as_string
else:
data = self.cleaned_data['data']
# Determine the data format
if self.cleaned_data['format'] == ImportFormatChoices.AUTO:
format = self._detect_format(data)
else:
format = self.cleaned_data['format']
# Process data according to the selected format
if format == ImportFormatChoices.CSV:
self.cleaned_data['data'] = self._clean_csv(data)
elif format == ImportFormatChoices.JSON:
self.cleaned_data['data'] = self._clean_json(data)
elif format == ImportFormatChoices.YAML:
self.cleaned_data['data'] = self._clean_yaml(data)
else:
raise forms.ValidationError(f"Unknown data format: {format}")
def _detect_format(self, data):
"""
Attempt to automatically detect the format (CSV, JSON, or YAML) of the given data, or raise
a ValidationError.
"""
try:
if data[0] in ('{', '['):
return ImportFormatChoices.JSON
if data.startswith('---') or data.startswith('- '):
return ImportFormatChoices.YAML
if ',' in data.split('\n', 1)[0]:
return ImportFormatChoices.CSV
except IndexError:
pass
raise forms.ValidationError({
'format': _('Unable to detect data format. Please specify.')
})
def _clean_csv(self, data):
"""
Clean CSV-formatted data. The first row will be treated as column headers.
"""
stream = StringIO(data.strip())
reader = csv.reader(stream)
headers, records = parse_csv(reader)
# Set CSV headers for reference by the model form
self._csv_headers = headers
return records
def _clean_json(self, data):
"""
Clean JSON-formatted data. If only a single object is defined, it will be encapsulated as a list.
"""
try:
data = json.loads(data)
# Accommodate for users entering single objects
if type(data) is not list:
data = [data]
return data
except json.decoder.JSONDecodeError as err:
raise forms.ValidationError({
self.data_field: f"Invalid JSON data: {err}"
})
def _clean_yaml(self, data):
"""
Clean YAML-formatted data. Data must be either
a) A single document comprising a list of dictionaries (each representing an object), or
b) Multiple documents, separated with the '---' token
"""
records = []
try:
for data in yaml.load_all(data, Loader=yaml.SafeLoader):
if type(data) == list:
records.extend(data)
elif type(data) == dict:
records.append(data)
else:
raise forms.ValidationError({
self.data_field: _(
"Invalid YAML data. Data must be in the form of multiple documents, or a single document "
"comprising a list of dictionaries."
)
})
except yaml.error.YAMLError as err:
raise forms.ValidationError({
self.data_field: f"Invalid YAML data: {err}"
})
return records

127
netbox/utilities/forms/forms.py
View File

@ -1,14 +1,8 @@
import csv
import json
import re
from io import StringIO
import yaml
from django import forms
from django.utils.translation import gettext as _
from utilities.choices import ImportFormatChoices
from utilities.forms.utils import parse_csv
from .widgets import APISelect, APISelectMultiple, ClearableFileInput
__all__ = (
@ -18,7 +12,6 @@ __all__ = (
'ConfirmationForm',
'CSVModelForm',
'FilterForm',
'ImportForm',
'ReturnURLForm',
'TableConfigForm',
)
@ -157,126 +150,6 @@ class CSVModelForm(forms.ModelForm):
del self.fields[field]
class ImportForm(BootstrapMixin, forms.Form):
data = forms.CharField(
required=False,
widget=forms.Textarea(attrs={'class': 'font-monospace'}),
help_text=_("Enter object data in CSV, JSON or YAML format.")
)
data_file = forms.FileField(
label="Data file",
required=False
)
format = forms.ChoiceField(
choices=ImportFormatChoices,
initial=ImportFormatChoices.AUTO
)
data_field = 'data'
def clean(self):
super().clean()
# Determine whether we're reading from form data or an uploaded file
if self.cleaned_data['data'] and self.cleaned_data['data_file']:
raise forms.ValidationError("Form data must be empty when uploading a file.")
if 'data_file' in self.files:
self.data_field = 'data_file'
file = self.files.get('data_file')
data = file.read().decode('utf-8-sig')
else:
data = self.cleaned_data['data']
# Determine the data format
if self.cleaned_data['format'] == ImportFormatChoices.AUTO:
format = self._detect_format(data)
else:
format = self.cleaned_data['format']
# Process data according to the selected format
if format == ImportFormatChoices.CSV:
self.cleaned_data['data'] = self._clean_csv(data)
elif format == ImportFormatChoices.JSON:
self.cleaned_data['data'] = self._clean_json(data)
elif format == ImportFormatChoices.YAML:
self.cleaned_data['data'] = self._clean_yaml(data)
else:
raise forms.ValidationError(f"Unknown data format: {format}")
def _detect_format(self, data):
"""
Attempt to automatically detect the format (CSV, JSON, or YAML) of the given data, or raise
a ValidationError.
"""
try:
if data[0] in ('{', '['):
return ImportFormatChoices.JSON
if data.startswith('---') or data.startswith('- '):
return ImportFormatChoices.YAML
if ',' in data.split('\n', 1)[0]:
return ImportFormatChoices.CSV
except IndexError:
pass
raise forms.ValidationError({
'format': _('Unable to detect data format. Please specify.')
})
def _clean_csv(self, data):
"""
Clean CSV-formatted data. The first row will be treated as column headers.
"""
stream = StringIO(data.strip())
reader = csv.reader(stream)
headers, records = parse_csv(reader)
# Set CSV headers for reference by the model form
self._csv_headers = headers
return records
def _clean_json(self, data):
"""
Clean JSON-formatted data. If only a single object is defined, it will be encapsulated as a list.
"""
try:
data = json.loads(data)
# Accommodate for users entering single objects
if type(data) is not list:
data = [data]
return data
except json.decoder.JSONDecodeError as err:
raise forms.ValidationError({
self.data_field: f"Invalid JSON data: {err}"
})
def _clean_yaml(self, data):
"""
Clean YAML-formatted data. Data must be either
a) A single document comprising a list of dictionaries (each representing an object), or
b) Multiple documents, separated with the '---' token
"""
records = []
try:
for data in yaml.load_all(data, Loader=yaml.SafeLoader):
if type(data) == list:
records.extend(data)
elif type(data) == dict:
records.append(data)
else:
raise forms.ValidationError({
self.data_field: _(
"Invalid YAML data. Data must be in the form of multiple documents, or a single document "
"comprising a list of dictionaries."
)
})
except yaml.error.YAMLError as err:
raise forms.ValidationError({
self.data_field: f"Invalid YAML data: {err}"
})
return records
class FilterForm(BootstrapMixin, forms.Form):
"""
Base Form class for FilterSet forms.

2
netbox/utilities/tests/test_forms.py
View File

@ -3,7 +3,7 @@ from django.test import TestCase
from ipam.forms import IPAddressImportForm
from utilities.choices import ImportFormatChoices
from utilities.forms import ImportForm
from utilities.forms.bulk_import import ImportForm
from utilities.forms.fields import CSVDataField
from utilities.forms.utils import expand_alphanumeric_pattern, expand_ipaddress_pattern