Closes #9073: Remote data support for config contexts (#11692)

* WIP

* Add bulk sync view for config contexts

* Introduce 'sync' permission for synced data models

* Docs & cleanup

* Remove unused method

* Add a REST API endpoint to synchronize config context data

netbox/netbox/api/features.py

@@ -0,0 +1,30 @@
+from django.shortcuts import get_object_or_404
+from rest_framework.decorators import action
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.response import Response
+
+from utilities.permissions import get_permission_for_model
+
+__all__ = (
+    'SyncedDataMixin',
+)
+
+
+class SyncedDataMixin:
+
+    @action(detail=True, methods=['post'])
+    def sync(self, request, pk):
+        """
+        Provide a /sync API endpoint to synchronize an object's data from its associated DataFile (if any).
+        """
+        permission = get_permission_for_model(self.queryset.model, 'sync')
+        if not request.user.has_perm(permission):
+            raise PermissionDenied(f"Missing permission: {permission}")
+
+        obj = get_object_or_404(self.queryset, pk=pk)
+        if obj.data_file:
+            obj.sync_data()
+            obj.save()
+        serializer = self.serializer_class(obj, context={'request': request})
+
+        return Response(serializer.data)

netbox/netbox/models/features.py

@@ -2,11 +2,12 @@ from collections import defaultdict
 from functools import cached_property
 
 from django.contrib.contenttypes.fields import GenericRelation
-from django.db.models.signals import class_prepared
-from django.dispatch import receiver
-
 from django.core.validators import ValidationError
 from django.db import models
+from django.db.models.signals import class_prepared
+from django.dispatch import receiver
+from django.utils.translation import gettext as _
+
 from taggit.managers import TaggableManager
 
 from extras.choices import CustomFieldVisibilityChoices, ObjectChangeActionChoices
@@ -25,6 +26,7 @@ __all__ = (
     'ExportTemplatesMixin',
     'JobResultsMixin',
     'JournalingMixin',
+    'SyncedDataMixin',
     'TagsMixin',
     'WebhooksMixin',
 )
@@ -317,12 +319,82 @@ class WebhooksMixin(models.Model):
         abstract = True
 
 
+class SyncedDataMixin(models.Model):
+    """
+    Enables population of local data from a DataFile object, synchronized from a remote DatSource.
+    """
+    data_source = models.ForeignKey(
+        to='core.DataSource',
+        on_delete=models.PROTECT,
+        blank=True,
+        null=True,
+        related_name='+',
+        help_text=_("Remote data source")
+    )
+    data_file = models.ForeignKey(
+        to='core.DataFile',
+        on_delete=models.SET_NULL,
+        blank=True,
+        null=True,
+        related_name='+'
+    )
+    data_path = models.CharField(
+        max_length=1000,
+        blank=True,
+        editable=False,
+        help_text=_("Path to remote file (relative to data source root)")
+    )
+    data_synced = models.DateTimeField(
+        blank=True,
+        null=True,
+        editable=False
+    )
+
+    class Meta:
+        abstract = True
+
+    @property
+    def is_synced(self):
+        return self.data_file and self.data_synced >= self.data_file.last_updated
+
+    def clean(self):
+        if self.data_file:
+            self.sync_data()
+            self.data_path = self.data_file.path
+
+        if self.data_source and not self.data_file:
+            raise ValidationError({
+                'data_file': _(f"Must specify a data file when designating a data source.")
+            })
+        if self.data_file and not self.data_source:
+            self.data_source = self.data_file.source
+
+        super().clean()
+
+    def resolve_data_file(self):
+        """
+        Determine the designated DataFile object identified by its parent DataSource and its path. Returns None if
+        either attribute is unset, or if no matching DataFile is found.
+        """
+        from core.models import DataFile
+
+        if self.data_source and self.data_path:
+            try:
+                return DataFile.objects.get(source=self.data_source, path=self.data_path)
+            except DataFile.DoesNotExist:
+                pass
+
+    def sync_data(self):
+        raise NotImplementedError(f"{self.__class__} must implement a sync_data() method.")
+
+
 FEATURES_MAP = (
     ('custom_fields', CustomFieldsMixin),
     ('custom_links', CustomLinksMixin),
     ('export_templates', ExportTemplatesMixin),
     ('job_results', JobResultsMixin),
     ('journaling', JournalingMixin),
+    ('synced_data', SyncedDataMixin),
     ('tags', TagsMixin),
     ('webhooks', WebhooksMixin),
 )
@@ -348,3 +420,9 @@ def _register_features(sender, **kwargs):
             'changelog',
             kwargs={'model': sender}
         )('netbox.views.generic.ObjectChangeLogView')
+    if issubclass(sender, SyncedDataMixin):
+        register_model_view(
+            sender,
+            'sync',
+            kwargs={'model': sender}
+        )('netbox.views.generic.ObjectSyncDataView')

netbox/netbox/views/generic/feature_views.py

@@ -1,16 +1,22 @@
 from django.contrib.contenttypes.models import ContentType
+from django.contrib import messages
+from django.db import transaction
 from django.db.models import Q
-from django.shortcuts import get_object_or_404, render
+from django.shortcuts import get_object_or_404, redirect, render
 from django.utils.translation import gettext as _
 from django.views.generic import View
 
 from extras import forms, tables
 from extras.models import *
-from utilities.views import ViewTab
+from utilities.permissions import get_permission_for_model
+from utilities.views import GetReturnURLMixin, ViewTab
+from .base import BaseMultiObjectView
 
 __all__ = (
+    'BulkSyncDataView',
     'ObjectChangeLogView',
     'ObjectJournalView',
+    'ObjectSyncDataView',
 )
 
 
@@ -126,3 +132,49 @@ class ObjectJournalView(View):
             'base_template': self.base_template,
             'tab': self.tab,
         })
+
+
+class ObjectSyncDataView(View):
+
+    def post(self, request, model, **kwargs):
+        """
+        Synchronize data from the DataFile associated with this object.
+        """
+        qs = model.objects.all()
+        if hasattr(model.objects, 'restrict'):
+            qs = qs.restrict(request.user, 'sync')
+        obj = get_object_or_404(qs, **kwargs)
+
+        if not obj.data_file:
+            messages.error(request, f"Unable to synchronize data: No data file set.")
+            return redirect(obj.get_absolute_url())
+
+        obj.sync_data()
+        obj.save()
+        messages.success(request, f"Synchronized data for {model._meta.verbose_name} {obj}.")
+
+        return redirect(obj.get_absolute_url())
+
+
+class BulkSyncDataView(GetReturnURLMixin, BaseMultiObjectView):
+    """
+    Synchronize multiple instances of a model inheriting from SyncedDataMixin.
+    """
+    def get_required_permission(self):
+        return get_permission_for_model(self.queryset.model, 'sync')
+
+    def post(self, request):
+        selected_objects = self.queryset.filter(
+            pk__in=request.POST.getlist('pk'),
+            data_file__isnull=False
+        )
+
+        with transaction.atomic():
+            for obj in selected_objects:
+                obj.sync_data()
+                obj.save()
+
+            model_name = self.queryset.model._meta.verbose_name_plural
+            messages.success(request, f"Synced {len(selected_objects)} {model_name}")
+
+        return redirect(self.get_return_url(request))