Closes #12129: Enable automatic synchronization of objects when DataFiles are updated (#12262)

* Closes #12129: Enable automatic synchronization of objects when DataFiles are updated

* Cleanup

netbox/core/forms/model_forms.py

@@ -80,12 +80,12 @@ class ManagedFileForm(SyncedDataMixin, NetBoxModelForm):
 
     fieldsets = (
         ('File Upload', ('upload_file',)),
-        ('Data Source', ('data_source', 'data_file')),
+        ('Data Source', ('data_source', 'data_file', 'auto_sync_enabled')),
     )
 
     class Meta:
         model = ManagedFile
-        fields = ('data_source', 'data_file')
+        fields = ('data_source', 'data_file', 'auto_sync_enabled')
 
     def clean(self):
         super().clean()

netbox/core/migrations/0001_initial.py

@@ -63,4 +63,21 @@ class Migration(migrations.Migration):
             model_name='datafile',
             index=models.Index(fields=['source', 'path'], name='core_datafile_source_path'),
         ),
+        migrations.CreateModel(
+            name='AutoSyncRecord',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)),
+                ('object_id', models.PositiveBigIntegerField()),
+                ('datafile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='core.datafile')),
+                ('object_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='contenttypes.contenttype')),
+            ],
+        ),
+        migrations.AddIndex(
+            model_name='autosyncrecord',
+            index=models.Index(fields=['object_type', 'object_id'], name='core_autosy_object__c17bac_idx'),
+        ),
+        migrations.AddConstraint(
+            model_name='autosyncrecord',
+            constraint=models.UniqueConstraint(fields=('object_type', 'object_id'), name='core_autosyncrecord_object'),
+        ),
     ]

netbox/core/migrations/0002_managedfile.py

@@ -23,6 +23,7 @@ class Migration(migrations.Migration):
                 ('file_path', models.FilePathField(editable=False)),
                 ('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')),
                 ('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')),
+                ('auto_sync_enabled', models.BooleanField(default=False)),
             ],
             options={
                 'ordering': ('file_root', 'file_path'),

netbox/core/models/data.py

@@ -5,7 +5,8 @@ from fnmatch import fnmatchcase
 from urllib.parse import urlparse
 
 from django.conf import settings
-from django.contrib.contenttypes.fields import GenericRelation
+from django.contrib.contenttypes.fields import GenericForeignKey
+from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
 from django.core.validators import RegexValidator
 from django.db import models
@@ -25,6 +26,7 @@ from ..signals import post_sync, pre_sync
 from .jobs import Job
 
 __all__ = (
+    'AutoSyncRecord',
     'DataFile',
     'DataSource',
 )
@@ -327,3 +329,35 @@ class DataFile(models.Model):
 
         with open(path, 'wb+') as new_file:
             new_file.write(self.data)
+
+
+class AutoSyncRecord(models.Model):
+    """
+    Maps a DataFile to a synced object for efficient automatic updating.
+    """
+    datafile = models.ForeignKey(
+        to=DataFile,
+        on_delete=models.CASCADE,
+        related_name='+'
+    )
+    object_type = models.ForeignKey(
+        to=ContentType,
+        on_delete=models.CASCADE,
+        related_name='+'
+    )
+    object_id = models.PositiveBigIntegerField()
+    object = GenericForeignKey(
+        ct_field='object_type',
+        fk_field='object_id'
+    )
+
+    class Meta:
+        constraints = (
+            models.UniqueConstraint(
+                fields=('object_type', 'object_id'),
+                name='%(app_label)s_%(class)s_object'
+            ),
+        )
+        indexes = (
+            models.Index(fields=('object_type', 'object_id')),
+        )

netbox/core/signals.py

@@ -1,4 +1,4 @@
-import django.dispatch
+from django.dispatch import Signal, receiver
 
 __all__ = (
     'post_sync',
@@ -6,5 +6,16 @@ __all__ = (
 )
 
 # DataSource signals
-pre_sync = django.dispatch.Signal()
+pre_sync = Signal()
-post_sync = django.dispatch.Signal()
+post_sync = Signal()
+
+
+@receiver(post_sync)
+def auto_sync(instance, **kwargs):
+    """
+    Automatically synchronize any DataFiles with AutoSyncRecords after synchronizing a DataSource.
+    """
+    from .models import AutoSyncRecord
+
+    for autosync in AutoSyncRecord.objects.filter(datafile__source=instance).prefetch_related('object'):
+        autosync.object.sync(save=True)

netbox/extras/forms/model_forms.py

@@ -112,7 +112,7 @@ class ExportTemplateForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm):
 
     fieldsets = (
         ('Export Template', ('name', 'content_types', 'description', 'template_code')),
-        ('Data Source', ('data_source', 'data_file')),
+        ('Data Source', ('data_source', 'data_file', 'auto_sync_enabled')),
         ('Rendering', ('mime_type', 'file_extension', 'as_attachment')),
     )
 
@@ -271,7 +271,7 @@ class ConfigContextForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm):
 
     fieldsets = (
         ('Config Context', ('name', 'weight', 'description', 'data', 'is_active')),
-        ('Data Source', ('data_source', 'data_file')),
+        ('Data Source', ('data_source', 'data_file', 'auto_sync_enabled')),
         ('Assignment', (
             'regions', 'site_groups', 'sites', 'locations', 'device_types', 'roles', 'platforms', 'cluster_types',
             'cluster_groups', 'clusters', 'tenant_groups', 'tenants', 'tags',
@@ -283,7 +283,7 @@ class ConfigContextForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm):
         fields = (
             'name', 'weight', 'description', 'data', 'is_active', 'regions', 'site_groups', 'sites', 'locations',
             'roles', 'device_types', 'platforms', 'cluster_types', 'cluster_groups', 'clusters', 'tenant_groups',
-            'tenants', 'tags', 'data_source', 'data_file',
+            'tenants', 'tags', 'data_source', 'data_file', 'auto_sync_enabled',
         )
 
     def __init__(self, *args, initial=None, **kwargs):
@@ -322,7 +322,7 @@ class ConfigTemplateForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm):
     fieldsets = (
         ('Config Template', ('name', 'description', 'environment_params', 'tags')),
         ('Content', ('template_code',)),
-        ('Data Source', ('data_source', 'data_file')),
+        ('Data Source', ('data_source', 'data_file', 'auto_sync_enabled')),
     )
 
     class Meta:

netbox/extras/migrations/0085_synced_data.py

@@ -26,6 +26,11 @@ class Migration(migrations.Migration):
             name='data_source',
             field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource'),
         ),
+        migrations.AddField(
+            model_name='configcontext',
+            name='auto_sync_enabled',
+            field=models.BooleanField(default=False),
+        ),
         migrations.AddField(
             model_name='configcontext',
             name='data_synced',
@@ -47,6 +52,11 @@ class Migration(migrations.Migration):
             name='data_source',
             field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource'),
         ),
+        migrations.AddField(
+            model_name='exporttemplate',
+            name='auto_sync_enabled',
+            field=models.BooleanField(default=False),
+        ),
         migrations.AddField(
             model_name='exporttemplate',
             name='data_synced',

netbox/extras/migrations/0086_configtemplate.py

@@ -25,6 +25,7 @@ class Migration(migrations.Migration):
                 ('environment_params', models.JSONField(blank=True, default=dict, null=True)),
                 ('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')),
                 ('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')),
+                ('auto_sync_enabled', models.BooleanField(default=False)),
                 ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')),
             ],
             options={

netbox/netbox/api/features.py

@@ -23,8 +23,7 @@ class SyncedDataMixin:
 
         obj = get_object_or_404(self.queryset, pk=pk)
         if obj.data_file:
-            obj.sync()
+            obj.sync(save=True)
-            obj.save()
         serializer = self.serializer_class(obj, context={'request': request})
 
         return Response(serializer.data)

netbox/netbox/models/features.py

@@ -3,6 +3,7 @@ from collections import defaultdict
 from functools import cached_property
 
 from django.contrib.contenttypes.fields import GenericRelation
+from django.contrib.contenttypes.models import ContentType
 from django.core.validators import ValidationError
 from django.db import models
 from django.db.models.signals import class_prepared
@@ -382,6 +383,10 @@ class SyncedDataMixin(models.Model):
         editable=False,
         help_text=_("Path to remote file (relative to data source root)")
     )
+    auto_sync_enabled = models.BooleanField(
+        default=False,
+        help_text=_("Enable automatic synchronization of data when the data file is updated")
+    )
     data_synced = models.DateTimeField(
         blank=True,
         null=True,
@@ -404,10 +409,33 @@ class SyncedDataMixin(models.Model):
         else:
             self.data_source = None
             self.data_path = ''
+            self.auto_sync_enabled = False
             self.data_synced = None
 
         super().clean()
 
+    def save(self, *args, **kwargs):
+        from core.models import AutoSyncRecord
+
+        ret = super().save(*args, **kwargs)
+
+        # Create/delete AutoSyncRecord as needed
+        content_type = ContentType.objects.get_for_model(self)
+        if self.auto_sync_enabled:
+            AutoSyncRecord.objects.get_or_create(
+                datafile=self.data_file,
+                object_type=content_type,
+                object_id=self.pk
+            )
+        else:
+            AutoSyncRecord.objects.filter(
+                datafile=self.data_file,
+                object_type=content_type,
+                object_id=self.pk
+            ).delete()
+
+        return ret
+
     def resolve_data_file(self):
         """
         Determine the designated DataFile object identified by its parent DataSource and its path. Returns None if
@@ -421,13 +449,17 @@ class SyncedDataMixin(models.Model):
             except DataFile.DoesNotExist:
                 pass
 
-    def sync(self):
+    def sync(self, save=False):
         """
         Synchronize the object from it's assigned DataFile (if any). This wraps sync_data() and updates
         the synced_data timestamp.
+
+        :param save: If true, save() will be called after data has been synchronized
         """
         self.sync_data()
         self.data_synced = timezone.now()
+        if save:
+            self.save()
 
     def sync_data(self):
         """

netbox/netbox/views/generic/feature_views.py

@@ -205,8 +205,7 @@ class ObjectSyncDataView(View):
             messages.error(request, f"Unable to synchronize data: No data file set.")
             return redirect(obj.get_absolute_url())
 
-        obj.sync()
+        obj.sync(save=True)
-        obj.save()
         messages.success(request, f"Synchronized data for {model._meta.verbose_name} {obj}.")
 
         return redirect(obj.get_absolute_url())
@@ -227,8 +226,7 @@ class BulkSyncDataView(GetReturnURLMixin, BaseMultiObjectView):
 
         with transaction.atomic():
             for obj in selected_objects:
-                obj.sync()
+                obj.sync(save=True)
-                obj.save()
 
             model_name = self.queryset.model._meta.verbose_name_plural
             messages.success(request, f"Synced {len(selected_objects)} {model_name}")