Closes #11890: Sync/upload reports & scripts (#12059)

* Initial work on #11890

* Consolidate get_scripts() and get_reports() functions

* Introduce proxy models for script & report modules

* Add add/delete views for reports & scripts

* Add deletion links for modules

* Enable resolving scripts/reports from module class

* Remove get_modules() utility function

* Show results in report/script lists

* Misc cleanup

* Fix file uploads

* Support automatic migration for submodules

* Fix module child ordering

* Template cleanup

* Remove ManagedFile views

* Move is_script(), is_report() into extras.utils

* Fix URLs for nested reports & scripts

* Misc cleanup

netbox/core/choices.py

@@ -20,7 +20,6 @@ class DataSourceTypeChoices(ChoiceSet):
 
 
 class DataSourceStatusChoices(ChoiceSet):
-
     NEW = 'new'
     QUEUED = 'queued'
     SYNCING = 'syncing'
@@ -34,3 +33,17 @@ class DataSourceStatusChoices(ChoiceSet):
         (COMPLETED, _('Completed'), 'green'),
         (FAILED, _('Failed'), 'red'),
     )
+
+
+#
+# Managed files
+#
+
+class ManagedFileRootPathChoices(ChoiceSet):
+    SCRIPTS = 'scripts'  # settings.SCRIPTS_ROOT
+    REPORTS = 'reports'  # settings.REPORTS_ROOT
+
+    CHOICES = (
+        (SCRIPTS, _('Scripts')),
+        (REPORTS, _('Reports')),
+    )

netbox/core/forms/model_forms.py

@@ -3,12 +3,14 @@ import copy
 from django import forms
 
 from core.models import *
+from extras.forms.mixins import SyncedDataMixin
 from netbox.forms import NetBoxModelForm
 from netbox.registry import registry
 from utilities.forms import CommentField, get_field_value
 
 __all__ = (
     'DataSourceForm',
+    'ManagedFileForm',
 )
 
 
@@ -73,3 +75,37 @@ class DataSourceForm(NetBoxModelForm):
         self.instance.parameters = parameters
 
         return super().save(*args, **kwargs)
+
+
+class ManagedFileForm(SyncedDataMixin, NetBoxModelForm):
+    upload_file = forms.FileField(
+        required=False
+    )
+
+    fieldsets = (
+        ('File Upload', ('upload_file',)),
+        ('Data Source', ('data_source', 'data_file')),
+    )
+
+    class Meta:
+        model = ManagedFile
+        fields = ('data_source', 'data_file')
+
+    def clean(self):
+        super().clean()
+
+        if self.cleaned_data.get('upload_file') and self.cleaned_data.get('data_file'):
+            raise forms.ValidationError("Cannot upload a file and sync from an existing file")
+        if not self.cleaned_data.get('upload_file') and not self.cleaned_data.get('data_file'):
+            raise forms.ValidationError("Must upload a file or select a data file to sync")
+
+        return self.cleaned_data
+
+    def save(self, *args, **kwargs):
+        # If a file was uploaded, save it to disk
+        if self.cleaned_data['upload_file']:
+            self.instance.file_path = self.cleaned_data['upload_file'].name
+            with open(self.instance.full_path, 'wb+') as new_file:
+                new_file.write(self.cleaned_data['upload_file'].read())
+
+        return super().save(*args, **kwargs)

netbox/core/migrations/0002_managedfile.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.1.7 on 2023-03-23 17:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ManagedFile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)),
+                ('data_path', models.CharField(blank=True, editable=False, max_length=1000)),
+                ('data_synced', models.DateTimeField(blank=True, editable=False, null=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('last_updated', models.DateTimeField(blank=True, editable=False, null=True)),
+                ('file_root', models.CharField(max_length=1000)),
+                ('file_path', models.FilePathField(editable=False)),
+                ('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')),
+                ('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')),
+            ],
+            options={
+                'ordering': ('file_root', 'file_path'),
+            },
+        ),
+        migrations.AddIndex(
+            model_name='managedfile',
+            index=models.Index(fields=['file_root', 'file_path'], name='core_managedfile_root_path'),
+        ),
+        migrations.AddConstraint(
+            model_name='managedfile',
+            constraint=models.UniqueConstraint(fields=('file_root', 'file_path'), name='core_managedfile_unique_root_path'),
+        ),
+    ]

netbox/core/models/__init__.py

@@ -1 +1,2 @@
 from .data import *
+from .files import *

netbox/core/models/data.py

@@ -14,7 +14,6 @@ from django.utils import timezone
 from django.utils.module_loading import import_string
 from django.utils.translation import gettext as _
 
-from extras.models import JobResult
 from netbox.models import PrimaryModel
 from netbox.registry import registry
 from utilities.files import sha256_hash
@@ -113,6 +112,8 @@ class DataSource(PrimaryModel):
         """
         Enqueue a background job to synchronize the DataSource by calling sync().
         """
+        from extras.models import JobResult
+
         # Set the status to "syncing"
         self.status = DataSourceStatusChoices.QUEUED
         DataSource.objects.filter(pk=self.pk).update(status=self.status)
@@ -314,3 +315,14 @@ class DataFile(models.Model):
                 self.data = f.read()
 
         return is_modified
+
+    def write_to_disk(self, path, overwrite=False):
+        """
+        Write the object's data to disk at the specified path
+        """
+        # Check whether file already exists
+        if os.path.isfile(path) and not overwrite:
+            raise FileExistsError()
+
+        with open(path, 'wb+') as new_file:
+            new_file.write(self.data)

netbox/core/models/files.py

@@ -0,0 +1,88 @@
+import logging
+import os
+
+from django.conf import settings
+from django.db import models
+from django.urls import reverse
+from django.utils.translation import gettext as _
+
+from ..choices import ManagedFileRootPathChoices
+from netbox.models.features import SyncedDataMixin
+from utilities.querysets import RestrictedQuerySet
+
+__all__ = (
+    'ManagedFile',
+)
+
+logger = logging.getLogger('netbox.core.files')
+
+
+class ManagedFile(SyncedDataMixin, models.Model):
+    """
+    Database representation for a file on disk. This class is typically wrapped by a proxy class (e.g. ScriptModule)
+    to provide additional functionality.
+    """
+    created = models.DateTimeField(
+        auto_now_add=True
+    )
+    last_updated = models.DateTimeField(
+        editable=False,
+        blank=True,
+        null=True
+    )
+    file_root = models.CharField(
+        max_length=1000,
+        choices=ManagedFileRootPathChoices
+    )
+    file_path = models.FilePathField(
+        editable=False,
+        help_text=_("File path relative to the designated root path")
+    )
+
+    objects = RestrictedQuerySet.as_manager()
+
+    class Meta:
+        ordering = ('file_root', 'file_path')
+        constraints = (
+            models.UniqueConstraint(
+                fields=('file_root', 'file_path'),
+                name='%(app_label)s_%(class)s_unique_root_path'
+            ),
+        )
+        indexes = [
+            models.Index(fields=('file_root', 'file_path'), name='core_managedfile_root_path'),
+        ]
+
+    def __str__(self):
+        return self.name
+
+    def get_absolute_url(self):
+        return reverse('core:managedfile', args=[self.pk])
+
+    @property
+    def name(self):
+        return self.file_path
+
+    @property
+    def full_path(self):
+        return os.path.join(self._resolve_root_path(), self.file_path)
+
+    def _resolve_root_path(self):
+        return {
+            'scripts': settings.SCRIPTS_ROOT,
+            'reports': settings.REPORTS_ROOT,
+        }[self.file_root]
+
+    def sync_data(self):
+        if self.data_file:
+            self.file_path = os.path.basename(self.data_path)
+            self.data_file.write_to_disk(self.full_path, overwrite=True)
+
+    def delete(self, *args, **kwargs):
+        # Delete file from disk
+        try:
+            os.remove(self.full_path)
+        except FileNotFoundError:
+            pass
+
+        return super().delete(*args, **kwargs)