Closes #11558: Add support for remote data sources (#11646)

* WIP

* WIP

* Add git sync

* Fix file hashing

* Add last_synced to DataSource

* Build out UI & API resources

* Add status field to DataSource

* Add UI control to sync data source

* Add API endpoint to sync data sources

* Fix display of DataSource job results

* DataSource password should be write-only

* General cleanup

* Add data file UI view

* Punt on HTTP, FTP support for now

* Add DataSource URL validation

* Add HTTP proxy support to git fetcher

* Add management command to sync data sources

* DataFile REST API endpoints should be read-only

* Refactor fetch methods into backend classes

* Replace auth & git branch fields with general-purpose parameters

* Fix last_synced time

* Render discrete form fields for backend parameters

* Enable dynamic edit form for DataSource

* Register DataBackend classes in application registry

* Add search indexers for DataSource, DataFile

* Add single & bulk delete views for DataFile

* Add model documentation

* Convert DataSource to a primary model

* Introduce pre_sync & post_sync signals

* Clean up migrations

* Rename url to source_url

* Clean up filtersets

* Add API & filterset tests

* Add view tests

* Add initSelect() to HTMX refresh handler

* Render DataSourceForm fieldsets dynamically

* Update compiled static resources

netbox/core/forms/__init__.py

@@ -0,0 +1,4 @@
+from .bulk_edit import *
+from .bulk_import import *
+from .filtersets import *
+from .model_forms import *

netbox/core/forms/bulk_edit.py

@@ -0,0 +1,50 @@
+from django import forms
+from django.utils.translation import gettext as _
+
+from core.choices import DataSourceTypeChoices
+from core.models import *
+from netbox.forms import NetBoxModelBulkEditForm
+from utilities.forms import (
+    add_blank_choice, BulkEditNullBooleanSelect, CommentField, SmallTextarea, StaticSelect,
+)
+
+__all__ = (
+    'DataSourceBulkEditForm',
+)
+
+
+class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
+    type = forms.ChoiceField(
+        choices=add_blank_choice(DataSourceTypeChoices),
+        required=False,
+        initial='',
+        widget=StaticSelect()
+    )
+    enabled = forms.NullBooleanField(
+        required=False,
+        widget=BulkEditNullBooleanSelect(),
+        label=_('Enforce unique space')
+    )
+    description = forms.CharField(
+        max_length=200,
+        required=False
+    )
+    comments = CommentField(
+        widget=SmallTextarea,
+        label=_('Comments')
+    )
+    parameters = forms.JSONField(
+        required=False
+    )
+    ignore_rules = forms.CharField(
+        required=False,
+        widget=forms.Textarea()
+    )
+
+    model = DataSource
+    fieldsets = (
+        (None, ('type', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules')),
+    )
+    nullable_fields = (
+        'description', 'description', 'parameters', 'comments', 'parameters', 'ignore_rules',
+    )

netbox/core/forms/bulk_import.py

@@ -0,0 +1,15 @@
+from core.models import *
+from netbox.forms import NetBoxModelImportForm
+
+__all__ = (
+    'DataSourceImportForm',
+)
+
+
+class DataSourceImportForm(NetBoxModelImportForm):
+
+    class Meta:
+        model = DataSource
+        fields = (
+            'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules',
+        )

netbox/core/forms/filtersets.py

@@ -0,0 +1,49 @@
+from django import forms
+from django.utils.translation import gettext as _
+
+from core.choices import *
+from core.models import *
+from netbox.forms import NetBoxModelFilterSetForm
+from utilities.forms import (
+    BOOLEAN_WITH_BLANK_CHOICES, DynamicModelMultipleChoiceField, MultipleChoiceField, StaticSelect,
+)
+
+__all__ = (
+    'DataFileFilterForm',
+    'DataSourceFilterForm',
+)
+
+
+class DataSourceFilterForm(NetBoxModelFilterSetForm):
+    model = DataSource
+    fieldsets = (
+        (None, ('q', 'filter_id')),
+        ('Data Source', ('type', 'status')),
+    )
+    type = MultipleChoiceField(
+        choices=DataSourceTypeChoices,
+        required=False
+    )
+    status = MultipleChoiceField(
+        choices=DataSourceStatusChoices,
+        required=False
+    )
+    enabled = forms.NullBooleanField(
+        required=False,
+        widget=StaticSelect(
+            choices=BOOLEAN_WITH_BLANK_CHOICES
+        )
+    )
+
+
+class DataFileFilterForm(NetBoxModelFilterSetForm):
+    model = DataFile
+    fieldsets = (
+        (None, ('q', 'filter_id')),
+        ('File', ('source_id',)),
+    )
+    source_id = DynamicModelMultipleChoiceField(
+        queryset=DataSource.objects.all(),
+        required=False,
+        label=_('Data source')
+    )

netbox/core/forms/model_forms.py

@@ -0,0 +1,81 @@
+import copy
+
+from django import forms
+
+from core.models import *
+from netbox.forms import NetBoxModelForm, StaticSelect
+from netbox.registry import registry
+from utilities.forms import CommentField
+
+__all__ = (
+    'DataSourceForm',
+)
+
+
+class DataSourceForm(NetBoxModelForm):
+    comments = CommentField()
+
+    class Meta:
+        model = DataSource
+        fields = [
+            'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
+        ]
+        widgets = {
+            'type': StaticSelect(
+                attrs={
+                    'hx-get': '.',
+                    'hx-include': '#form_fields input',
+                    'hx-target': '#form_fields',
+                }
+            ),
+            'ignore_rules': forms.Textarea(
+                attrs={
+                    'rows': 5,
+                    'class': 'font-monospace',
+                    'placeholder': '.cache\n*.txt'
+                }
+            ),
+        }
+
+    @property
+    def fieldsets(self):
+        fieldsets = [
+            ('Source', ('name', 'type', 'source_url', 'enabled', 'description', 'tags', 'ignore_rules')),
+        ]
+        if self.backend_fields:
+            fieldsets.append(
+                ('Backend', self.backend_fields)
+            )
+
+        return fieldsets
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        backend_classes = registry['data_backends']
+
+        if self.is_bound and self.data.get('type') in backend_classes:
+            type_ = self.data['type']
+        elif self.initial and self.initial.get('type') in backend_classes:
+            type_ = self.initial['type']
+        else:
+            type_ = self.fields['type'].initial
+        backend = backend_classes.get(type_)
+
+        self.backend_fields = []
+        for name, form_field in backend.parameters.items():
+            field_name = f'backend_{name}'
+            self.backend_fields.append(field_name)
+            self.fields[field_name] = copy.copy(form_field)
+            if self.instance and self.instance.parameters:
+                self.fields[field_name].initial = self.instance.parameters.get(name)
+
+    def save(self, *args, **kwargs):
+
+        parameters = {}
+        for name in self.fields:
+            if name.startswith('backend_'):
+                parameters[name[8:]] = self.cleaned_data[name]
+        self.instance.parameters = parameters
+
+        return super().save(*args, **kwargs)