Closes #11558: Add support for remote data sources (#11646)

* WIP * WIP * Add git sync * Fix file hashing * Add last_synced to DataSource * Build out UI & API resources * Add status field to DataSource * Add UI control to sync data source * Add API endpoint to sync data sources * Fix display of DataSource job results * DataSource password should be write-only * General cleanup * Add data file UI view * Punt on HTTP, FTP support for now * Add DataSource URL validation * Add HTTP proxy support to git fetcher * Add management command to sync data sources * DataFile REST API endpoints should be read-only * Refactor fetch methods into backend classes * Replace auth & git branch fields with general-purpose parameters * Fix last_synced time * Render discrete form fields for backend parameters * Enable dynamic edit form for DataSource * Register DataBackend classes in application registry * Add search indexers for DataSource, DataFile * Add single & bulk delete views for DataFile * Add model documentation * Convert DataSource to a primary model * Introduce pre_sync & post_sync signals * Clean up migrations * Rename url to source_url * Clean up filtersets * Add API & filterset tests * Add view tests * Add initSelect() to HTMX refresh handler * Render DataSourceForm fieldsets dynamically * Update compiled static resources
2024-05-10 07:54:54 +00:00 · 2023-02-02 10:06:23 -05:00
parent e65b2a9fb3
commit d8784d4155
53 changed files with 1865 additions and 14 deletions
--- a/netbox/core/tests/test_views.py
+++ b/netbox/core/tests/test_views.py
@@ -0,0 +1,91 @@
+from django.utils import timezone
+
+from utilities.testing import ViewTestCases, create_tags
+from ..choices import *
+from ..models import *
+
+
+class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
+    model = DataSource
+
+    @classmethod
+    def setUpTestData(cls):
+        data_sources = (
+            DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'),
+            DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'),
+            DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'),
+        )
+        DataSource.objects.bulk_create(data_sources)
+
+        tags = create_tags('Alpha', 'Bravo', 'Charlie')
+
+        cls.form_data = {
+            'name': 'Data Source X',
+            'type': DataSourceTypeChoices.GIT,
+            'source_url': 'http:///exmaple/com/foo/bar/',
+            'description': 'Something',
+            'comments': 'Foo bar baz',
+            'tags': [t.pk for t in tags],
+        }
+
+        cls.csv_data = (
+            f"name,type,source_url,enabled",
+            f"Data Source 4,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true",
+            f"Data Source 5,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true",
+            f"Data Source 6,{DataSourceTypeChoices.GIT},http:///exmaple/com/foo/bar/,false",
+        )
+
+        cls.csv_update_data = (
+            "id,name,description",
+            f"{data_sources[0].pk},Data Source 7,New description7",
+            f"{data_sources[1].pk},Data Source 8,New description8",
+            f"{data_sources[2].pk},Data Source 9,New description9",
+        )
+
+        cls.bulk_edit_data = {
+            'enabled': False,
+            'description': 'New description',
+        }
+
+
+class DataFileTestCase(
+    ViewTestCases.GetObjectViewTestCase,
+    ViewTestCases.GetObjectChangelogViewTestCase,
+    ViewTestCases.DeleteObjectViewTestCase,
+    ViewTestCases.ListObjectsViewTestCase,
+    ViewTestCases.BulkDeleteObjectsViewTestCase,
+):
+    model = DataFile
+
+    @classmethod
+    def setUpTestData(cls):
+        datasource = DataSource.objects.create(
+            name='Data Source 1',
+            type=DataSourceTypeChoices.LOCAL,
+            source_url='file:///var/tmp/source1/'
+        )
+
+        data_files = (
+            DataFile(
+                source=datasource,
+                path='dir1/file1.txt',
+                last_updated=timezone.now(),
+                size=1000,
+                hash='442da078f0111cbdf42f21903724f6597c692535f55bdfbbea758a1ae99ad9e1'
+            ),
+            DataFile(
+                source=datasource,
+                path='dir1/file2.txt',
+                last_updated=timezone.now(),
+                size=2000,
+                hash='a78168c7c97115bafd96450ed03ea43acec495094c5caa28f0d02e20e3a76cc2'
+            ),
+            DataFile(
+                source=datasource,
+                path='dir1/file3.txt',
+                last_updated=timezone.now(),
+                size=3000,
+                hash='12b8827a14c4d5a2f30b6c6e2b7983063988612391c6cbe8ee7493b59054827a'
+            ),
+        )
+        DataFile.objects.bulk_create(data_files)