diff --git a/netbox/core/forms/model_forms.py b/netbox/core/forms/model_forms.py index 304bc346a..405d70437 100644 --- a/netbox/core/forms/model_forms.py +++ b/netbox/core/forms/model_forms.py @@ -80,12 +80,12 @@ class ManagedFileForm(SyncedDataMixin, NetBoxModelForm): fieldsets = ( ('File Upload', ('upload_file',)), - ('Data Source', ('data_source', 'data_file')), + ('Data Source', ('data_source', 'data_file', 'auto_sync_enabled')), ) class Meta: model = ManagedFile - fields = ('data_source', 'data_file') + fields = ('data_source', 'data_file', 'auto_sync_enabled') def clean(self): super().clean() diff --git a/netbox/core/migrations/0001_initial.py b/netbox/core/migrations/0001_initial.py index 0678d4a67..775a5dcb1 100644 --- a/netbox/core/migrations/0001_initial.py +++ b/netbox/core/migrations/0001_initial.py @@ -63,4 +63,21 @@ class Migration(migrations.Migration): model_name='datafile', index=models.Index(fields=['source', 'path'], name='core_datafile_source_path'), ), + migrations.CreateModel( + name='AutoSyncRecord', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('object_id', models.PositiveBigIntegerField()), + ('datafile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='core.datafile')), + ('object_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='+', to='contenttypes.contenttype')), + ], + ), + migrations.AddIndex( + model_name='autosyncrecord', + index=models.Index(fields=['object_type', 'object_id'], name='core_autosy_object__c17bac_idx'), + ), + migrations.AddConstraint( + model_name='autosyncrecord', + constraint=models.UniqueConstraint(fields=('object_type', 'object_id'), name='core_autosyncrecord_object'), + ), ] diff --git a/netbox/core/migrations/0002_managedfile.py b/netbox/core/migrations/0002_managedfile.py index da6b1e3be..169063be8 100644 --- a/netbox/core/migrations/0002_managedfile.py +++ b/netbox/core/migrations/0002_managedfile.py @@ -23,6 +23,7 @@ class Migration(migrations.Migration): ('file_path', models.FilePathField(editable=False)), ('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')), ('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')), + ('auto_sync_enabled', models.BooleanField(default=False)), ], options={ 'ordering': ('file_root', 'file_path'), diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index 1738190ee..efd98a7b9 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -5,7 +5,8 @@ from fnmatch import fnmatchcase from urllib.parse import urlparse from django.conf import settings -from django.contrib.contenttypes.fields import GenericRelation +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError from django.core.validators import RegexValidator from django.db import models @@ -25,6 +26,7 @@ from ..signals import post_sync, pre_sync from .jobs import Job __all__ = ( + 'AutoSyncRecord', 'DataFile', 'DataSource', ) @@ -327,3 +329,35 @@ class DataFile(models.Model): with open(path, 'wb+') as new_file: new_file.write(self.data) + + +class AutoSyncRecord(models.Model): + """ + Maps a DataFile to a synced object for efficient automatic updating. + """ + datafile = models.ForeignKey( + to=DataFile, + on_delete=models.CASCADE, + related_name='+' + ) + object_type = models.ForeignKey( + to=ContentType, + on_delete=models.CASCADE, + related_name='+' + ) + object_id = models.PositiveBigIntegerField() + object = GenericForeignKey( + ct_field='object_type', + fk_field='object_id' + ) + + class Meta: + constraints = ( + models.UniqueConstraint( + fields=('object_type', 'object_id'), + name='%(app_label)s_%(class)s_object' + ), + ) + indexes = ( + models.Index(fields=('object_type', 'object_id')), + ) diff --git a/netbox/core/signals.py b/netbox/core/signals.py index 65ca293f5..a39a87c6a 100644 --- a/netbox/core/signals.py +++ b/netbox/core/signals.py @@ -1,4 +1,4 @@ -import django.dispatch +from django.dispatch import Signal, receiver __all__ = ( 'post_sync', @@ -6,5 +6,16 @@ __all__ = ( ) # DataSource signals -pre_sync = django.dispatch.Signal() -post_sync = django.dispatch.Signal() +pre_sync = Signal() +post_sync = Signal() + + +@receiver(post_sync) +def auto_sync(instance, **kwargs): + """ + Automatically synchronize any DataFiles with AutoSyncRecords after synchronizing a DataSource. + """ + from .models import AutoSyncRecord + + for autosync in AutoSyncRecord.objects.filter(datafile__source=instance).prefetch_related('object'): + autosync.object.sync(save=True) diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index c7c55e282..bc79d8fe5 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -112,7 +112,7 @@ class ExportTemplateForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm): fieldsets = ( ('Export Template', ('name', 'content_types', 'description', 'template_code')), - ('Data Source', ('data_source', 'data_file')), + ('Data Source', ('data_source', 'data_file', 'auto_sync_enabled')), ('Rendering', ('mime_type', 'file_extension', 'as_attachment')), ) @@ -271,7 +271,7 @@ class ConfigContextForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm): fieldsets = ( ('Config Context', ('name', 'weight', 'description', 'data', 'is_active')), - ('Data Source', ('data_source', 'data_file')), + ('Data Source', ('data_source', 'data_file', 'auto_sync_enabled')), ('Assignment', ( 'regions', 'site_groups', 'sites', 'locations', 'device_types', 'roles', 'platforms', 'cluster_types', 'cluster_groups', 'clusters', 'tenant_groups', 'tenants', 'tags', @@ -283,7 +283,7 @@ class ConfigContextForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm): fields = ( 'name', 'weight', 'description', 'data', 'is_active', 'regions', 'site_groups', 'sites', 'locations', 'roles', 'device_types', 'platforms', 'cluster_types', 'cluster_groups', 'clusters', 'tenant_groups', - 'tenants', 'tags', 'data_source', 'data_file', + 'tenants', 'tags', 'data_source', 'data_file', 'auto_sync_enabled', ) def __init__(self, *args, initial=None, **kwargs): @@ -322,7 +322,7 @@ class ConfigTemplateForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm): fieldsets = ( ('Config Template', ('name', 'description', 'environment_params', 'tags')), ('Content', ('template_code',)), - ('Data Source', ('data_source', 'data_file')), + ('Data Source', ('data_source', 'data_file', 'auto_sync_enabled')), ) class Meta: diff --git a/netbox/extras/migrations/0085_synced_data.py b/netbox/extras/migrations/0085_synced_data.py index 4790cd51a..372845490 100644 --- a/netbox/extras/migrations/0085_synced_data.py +++ b/netbox/extras/migrations/0085_synced_data.py @@ -26,6 +26,11 @@ class Migration(migrations.Migration): name='data_source', field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource'), ), + migrations.AddField( + model_name='configcontext', + name='auto_sync_enabled', + field=models.BooleanField(default=False), + ), migrations.AddField( model_name='configcontext', name='data_synced', @@ -47,6 +52,11 @@ class Migration(migrations.Migration): name='data_source', field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource'), ), + migrations.AddField( + model_name='exporttemplate', + name='auto_sync_enabled', + field=models.BooleanField(default=False), + ), migrations.AddField( model_name='exporttemplate', name='data_synced', diff --git a/netbox/extras/migrations/0086_configtemplate.py b/netbox/extras/migrations/0086_configtemplate.py index 82f2b38a3..32d4e9858 100644 --- a/netbox/extras/migrations/0086_configtemplate.py +++ b/netbox/extras/migrations/0086_configtemplate.py @@ -25,6 +25,7 @@ class Migration(migrations.Migration): ('environment_params', models.JSONField(blank=True, default=dict, null=True)), ('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')), ('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')), + ('auto_sync_enabled', models.BooleanField(default=False)), ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')), ], options={ diff --git a/netbox/netbox/api/features.py b/netbox/netbox/api/features.py index 3c2a096c6..9a83089ee 100644 --- a/netbox/netbox/api/features.py +++ b/netbox/netbox/api/features.py @@ -23,8 +23,7 @@ class SyncedDataMixin: obj = get_object_or_404(self.queryset, pk=pk) if obj.data_file: - obj.sync() - obj.save() + obj.sync(save=True) serializer = self.serializer_class(obj, context={'request': request}) return Response(serializer.data) diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index cf141d987..6d82e2a2b 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -3,6 +3,7 @@ from collections import defaultdict from functools import cached_property from django.contrib.contenttypes.fields import GenericRelation +from django.contrib.contenttypes.models import ContentType from django.core.validators import ValidationError from django.db import models from django.db.models.signals import class_prepared @@ -382,6 +383,10 @@ class SyncedDataMixin(models.Model): editable=False, help_text=_("Path to remote file (relative to data source root)") ) + auto_sync_enabled = models.BooleanField( + default=False, + help_text=_("Enable automatic synchronization of data when the data file is updated") + ) data_synced = models.DateTimeField( blank=True, null=True, @@ -404,10 +409,33 @@ class SyncedDataMixin(models.Model): else: self.data_source = None self.data_path = '' + self.auto_sync_enabled = False self.data_synced = None super().clean() + def save(self, *args, **kwargs): + from core.models import AutoSyncRecord + + ret = super().save(*args, **kwargs) + + # Create/delete AutoSyncRecord as needed + content_type = ContentType.objects.get_for_model(self) + if self.auto_sync_enabled: + AutoSyncRecord.objects.get_or_create( + datafile=self.data_file, + object_type=content_type, + object_id=self.pk + ) + else: + AutoSyncRecord.objects.filter( + datafile=self.data_file, + object_type=content_type, + object_id=self.pk + ).delete() + + return ret + def resolve_data_file(self): """ Determine the designated DataFile object identified by its parent DataSource and its path. Returns None if @@ -421,13 +449,17 @@ class SyncedDataMixin(models.Model): except DataFile.DoesNotExist: pass - def sync(self): + def sync(self, save=False): """ Synchronize the object from it's assigned DataFile (if any). This wraps sync_data() and updates the synced_data timestamp. + + :param save: If true, save() will be called after data has been synchronized """ self.sync_data() self.data_synced = timezone.now() + if save: + self.save() def sync_data(self): """ diff --git a/netbox/netbox/views/generic/feature_views.py b/netbox/netbox/views/generic/feature_views.py index d78c8f754..95b7b5712 100644 --- a/netbox/netbox/views/generic/feature_views.py +++ b/netbox/netbox/views/generic/feature_views.py @@ -205,8 +205,7 @@ class ObjectSyncDataView(View): messages.error(request, f"Unable to synchronize data: No data file set.") return redirect(obj.get_absolute_url()) - obj.sync() - obj.save() + obj.sync(save=True) messages.success(request, f"Synchronized data for {model._meta.verbose_name} {obj}.") return redirect(obj.get_absolute_url()) @@ -227,8 +226,7 @@ class BulkSyncDataView(GetReturnURLMixin, BaseMultiObjectView): with transaction.atomic(): for obj in selected_objects: - obj.sync() - obj.save() + obj.sync(save=True) model_name = self.queryset.model._meta.verbose_name_plural messages.success(request, f"Synced {len(selected_objects)} {model_name}")