Closes #11693: Enable remote data synchronization for export templates

docs/models/extras/exporttemplate.md

@@ -12,6 +12,10 @@ The name of the export template. This will appear in the "export" dropdown list
 
 The type of NetBox object to which the export template applies.
 
+### Data File
+
+Template code may optionally be sourced from a remote [data file](../core/datafile.md), which is synchronized from a remote data source. When designating a data file, there is no need to specify local content for the template: It will be populated automatically from the data file.
+
 ### Template Code
 
 Jinja2 template code for rendering the exported data.

netbox/extras/api/serializers.py

@@ -142,12 +142,19 @@ class ExportTemplateSerializer(ValidatedModelSerializer):
         queryset=ContentType.objects.filter(FeatureQuery('export_templates').get_query()),
         many=True
     )
+    data_source = NestedDataSourceSerializer(
+        required=False
+    )
+    data_file = NestedDataFileSerializer(
+        read_only=True
+    )
 
     class Meta:
         model = ExportTemplate
         fields = [
             'id', 'url', 'display', 'content_types', 'name', 'description', 'template_code', 'mime_type',
-            'file_extension', 'as_attachment', 'created', 'last_updated',
+            'file_extension', 'as_attachment', 'data_source', 'data_path', 'data_file', 'data_synced', 'created',
+            'last_updated',
         ]
 
 

netbox/extras/api/views.py

@@ -92,9 +92,9 @@ class CustomLinkViewSet(NetBoxModelViewSet):
 # Export templates
 #
 
-class ExportTemplateViewSet(NetBoxModelViewSet):
+class ExportTemplateViewSet(SyncedDataMixin, NetBoxModelViewSet):
     metadata_class = ContentTypeMetadata
-    queryset = ExportTemplate.objects.all()
+    queryset = ExportTemplate.objects.prefetch_related('data_source', 'data_file')
     serializer_class = serializers.ExportTemplateSerializer
     filterset_class = filtersets.ExportTemplateFilterSet
 

netbox/extras/filtersets.py

@@ -127,10 +127,18 @@ class ExportTemplateFilterSet(BaseFilterSet):
         field_name='content_types__id'
     )
     content_types = ContentTypeFilter()
+    data_source_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=DataSource.objects.all(),
+        label=_('Data source (ID)'),
+    )
+    data_file_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=DataSource.objects.all(),
+        label=_('Data file (ID)'),
+    )
 
     class Meta:
         model = ExportTemplate
-        fields = ['id', 'content_types', 'name', 'description']
+        fields = ['id', 'content_types', 'name', 'description', 'data_synced']
 
     def search(self, queryset, name, value):
         if not value.strip():

netbox/extras/forms/filtersets.py

@@ -21,9 +21,9 @@ from .mixins import SavedFiltersMixin
 __all__ = (
     'ConfigContextFilterForm',
     'CustomFieldFilterForm',
-    'JobResultFilterForm',
     'CustomLinkFilterForm',
     'ExportTemplateFilterForm',
+    'JobResultFilterForm',
     'JournalEntryFilterForm',
     'LocalConfigContextFilterForm',
     'ObjectChangeFilterForm',
@@ -157,8 +157,22 @@ class CustomLinkFilterForm(SavedFiltersMixin, FilterForm):
 class ExportTemplateFilterForm(SavedFiltersMixin, FilterForm):
     fieldsets = (
         (None, ('q', 'filter_id')),
+        ('Data', ('data_source_id', 'data_file_id')),
         ('Attributes', ('content_types', 'mime_type', 'file_extension', 'as_attachment')),
     )
+    data_source_id = DynamicModelMultipleChoiceField(
+        queryset=DataSource.objects.all(),
+        required=False,
+        label=_('Data source')
+    )
+    data_file_id = DynamicModelMultipleChoiceField(
+        queryset=DataFile.objects.all(),
+        required=False,
+        label=_('Data file'),
+        query_params={
+            'source_id': '$data_source_id'
+        }
+    )
     content_types = ContentTypeMultipleChoiceField(
         queryset=ContentType.objects.filter(FeatureQuery('export_templates').get_query()),
         required=False

netbox/extras/forms/model_forms.py

@@ -96,19 +96,28 @@ class ExportTemplateForm(BootstrapMixin, forms.ModelForm):
         queryset=ContentType.objects.all(),
         limit_choices_to=FeatureQuery('export_templates')
     )
+    template_code = forms.CharField(
+        required=False,
+        widget=forms.Textarea(attrs={'class': 'font-monospace'})
+    )
 
     fieldsets = (
         ('Export Template', ('name', 'content_types', 'description')),
-        ('Template', ('template_code',)),
+        ('Content', ('data_source', 'data_file', 'template_code',)),
         ('Rendering', ('mime_type', 'file_extension', 'as_attachment')),
     )
 
     class Meta:
         model = ExportTemplate
         fields = '__all__'
-        widgets = {
-            'template_code': forms.Textarea(attrs={'class': 'font-monospace'}),
-        }
+
+    def clean(self):
+        super().clean()
+
+        if not self.cleaned_data.get('template_code') and not self.cleaned_data.get('data_file'):
+            raise forms.ValidationError("Must specify either local content or a data file")
+
+        return self.cleaned_data
 
 
 class SavedFilterForm(BootstrapMixin, forms.ModelForm):
@@ -261,8 +270,8 @@ class ConfigContextForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm):
     def clean(self):
         super().clean()
 
-        if not self.cleaned_data.get('data') and not self.cleaned_data.get('data_source'):
-            raise forms.ValidationError("Must specify either local data or a data source")
+        if not self.cleaned_data.get('data') and not self.cleaned_data.get('data_file'):
+            raise forms.ValidationError("Must specify either local data or a data file")
 
         return self.cleaned_data
 

netbox/extras/migrations/0086_exporttemplate_synced_data.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.1.6 on 2023-02-08 22:16
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+        ('extras', '0085_configcontext_synced_data'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='exporttemplate',
+            name='data_file',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile'),
+        ),
+        migrations.AddField(
+            model_name='exporttemplate',
+            name='data_path',
+            field=models.CharField(blank=True, editable=False, max_length=1000),
+        ),
+        migrations.AddField(
+            model_name='exporttemplate',
+            name='data_source',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource'),
+        ),
+        migrations.AddField(
+            model_name='exporttemplate',
+            name='data_synced',
+            field=models.DateTimeField(blank=True, editable=False, null=True),
+        ),
+    ]

netbox/extras/models/models.py

@@ -26,7 +26,8 @@ from netbox.config import get_config
 from netbox.constants import RQ_QUEUE_DEFAULT
 from netbox.models import ChangeLoggedModel
 from netbox.models.features import (
-    CloningMixin, CustomFieldsMixin, CustomLinksMixin, ExportTemplatesMixin, JobResultsMixin, TagsMixin, WebhooksMixin,
+    CloningMixin, CustomFieldsMixin, CustomLinksMixin, ExportTemplatesMixin, JobResultsMixin, SyncedDataMixin,
+    TagsMixin, WebhooksMixin,
 )
 from utilities.querysets import RestrictedQuerySet
 from utilities.utils import render_jinja2
@@ -281,7 +282,7 @@ class CustomLink(CloningMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLogged
         }
 
 
-class ExportTemplate(ExportTemplatesMixin, WebhooksMixin, ChangeLoggedModel):
+class ExportTemplate(SyncedDataMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLoggedModel):
     content_types = models.ManyToManyField(
         to=ContentType,
         related_name='export_templates',
@@ -335,6 +336,13 @@ class ExportTemplate(ExportTemplatesMixin, WebhooksMixin, ChangeLoggedModel):
                 'name': f'"{self.name}" is a reserved name. Please choose a different name.'
             })
 
+    def sync_data(self):
+        """
+        Synchronize template content from the designated DataFile (if any).
+        """
+        self.template_code = self.data_file.data_as_string
+        self.data_synced = timezone.now()
+
     def render(self, queryset):
         """
         Render the contents of the template.

netbox/extras/tables/tables.py

@@ -90,15 +90,24 @@ class ExportTemplateTable(NetBoxTable):
     )
     content_types = columns.ContentTypesColumn()
     as_attachment = columns.BooleanColumn()
+    data_source = tables.Column(
+        linkify=True
+    )
+    data_file = tables.Column(
+        linkify=True
+    )
+    is_synced = columns.BooleanColumn(
+        verbose_name='Synced'
+    )
 
     class Meta(NetBoxTable.Meta):
         model = ExportTemplate
         fields = (
             'pk', 'id', 'name', 'content_types', 'description', 'mime_type', 'file_extension', 'as_attachment',
-            'created', 'last_updated',
+            'data_source', 'data_file', 'data_synced', 'created', 'last_updated',
         )
         default_columns = (
-            'pk', 'name', 'content_types', 'description', 'mime_type', 'file_extension', 'as_attachment',
+            'pk', 'name', 'content_types', 'description', 'mime_type', 'file_extension', 'as_attachment', 'is_synced',
         )
 
 

netbox/extras/urls.py

@@ -29,6 +29,7 @@ urlpatterns = [
     path('export-templates/import/', views.ExportTemplateBulkImportView.as_view(), name='exporttemplate_import'),
     path('export-templates/edit/', views.ExportTemplateBulkEditView.as_view(), name='exporttemplate_bulk_edit'),
     path('export-templates/delete/', views.ExportTemplateBulkDeleteView.as_view(), name='exporttemplate_bulk_delete'),
+    path('export-templates/sync/', views.ExportTemplateBulkSyncDataView.as_view(), name='exporttemplate_bulk_sync'),
     path('export-templates/<int:pk>/', include(get_model_urls('extras', 'exporttemplate'))),
 
     # Saved filters

netbox/extras/views.py

@@ -121,6 +121,8 @@ class ExportTemplateListView(generic.ObjectListView):
     filterset = filtersets.ExportTemplateFilterSet
     filterset_form = forms.ExportTemplateFilterForm
     table = tables.ExportTemplateTable
+    template_name = 'extras/exporttemplate_list.html'
+    actions = ('add', 'import', 'export', 'bulk_edit', 'bulk_delete', 'bulk_sync')
 
 
 @register_model_view(ExportTemplate)
@@ -158,6 +160,10 @@ class ExportTemplateBulkDeleteView(generic.BulkDeleteView):
     table = tables.ExportTemplateTable
 
 
+class ExportTemplateBulkSyncDataView(generic.BulkSyncDataView):
+    queryset = ExportTemplate.objects.all()
+
+
 #
 # Saved filters
 #

netbox/templates/extras/configcontext.html

@@ -50,10 +50,10 @@
                 {% endif %}
               </td>
             </tr>
-          <tr>
-            <th scope="row">Data Synced</th>
-            <td>{{ object.data_synced|placeholder }}</td>
-          </tr>
+            <tr>
+              <th scope="row">Data Synced</th>
+              <td>{{ object.data_synced|placeholder }}</td>
+            </tr>
           </table>
         </div>
       </div>
@@ -86,22 +86,8 @@
           {% include 'extras/inc/configcontext_format.html' %}
         </div>
         <div class="card-body">
-          {% if object.data_file and object.data_file.last_updated > object.data_synced %}
-            <div class="alert alert-warning" role="alert">
-              <i class="mdi mdi-alert"></i> Data is out of sync with upstream file (<a href="{{ object.data_file.get_absolute_url }}">{{ object.data_file }}</a>).
-              {% if perms.extras.sync_configcontext %}
-                <div class="float-end">
-                  <form action="{% url 'extras:configcontext_sync' pk=object.pk %}" method="post">
-                    {% csrf_token %}
-                    <button type="submit" class="btn btn-primary btn-sm">
-                      <i class="mdi mdi-sync" aria-hidden="true"></i> Sync
-                    </button>
-                  </form>
-                </div>
-              {% endif %}
-            </div>
-          {% endif %}
-            {% include 'extras/inc/configcontext_data.html' with data=object.data format=format %}
+          {% include 'inc/sync_warning.html' %}
+          {% include 'extras/inc/configcontext_data.html' with data=object.data format=format %}
         </div>
       </div>
     </div>

netbox/templates/extras/exporttemplate.html

@@ -10,66 +10,92 @@
 {% endblock %}
 
 {% block content %}
-<div class="row mb-3">
-	<div class="col col-md-5">
-    <div class="card">
-      <h5 class="card-header">
-        Export Template
-      </h5>
-      <div class="card-body">
-        <table class="table table-hover attr-table">
-          <tr>
-            <th scope="row">Name</th>
-            <td>{{ object.name }}</td>
-          </tr>
-          <tr>
-            <th scope="row">Description</th>
-            <td>{{ object.description|placeholder }}</td>
-          </tr>
-          <tr>
-            <th scope="row">MIME Type</th>
-            <td>{{ object.mime_type|placeholder }}</td>
-          </tr>
-          <tr>
-            <th scope="row">File Extension</th>
-            <td>{{ object.file_extension|placeholder }}</td>
-          </tr>
-          <tr>
-            <th scope="row">Attachment</th>
-            <td>{% checkmark object.as_attachment %}</td>
-          </tr>
-        </table>
-      </div>
-    </div>
-    <div class="card">
-      <h5 class="card-header">Assigned Models</h5>
-      <div class="card-body">
-        <table class="table table-hover attr-table">
-          {% for ct in object.content_types.all %}
+  <div class="row mb-3">
+    <div class="col col-md-5">
+      <div class="card">
+        <h5 class="card-header">Export Template</h5>
+        <div class="card-body">
+          <table class="table table-hover attr-table">
             <tr>
-              <td>{{ ct }}</td>
+              <th scope="row">Name</th>
+              <td>{{ object.name }}</td>
             </tr>
-          {% endfor %}
-        </table>
+            <tr>
+              <th scope="row">Description</th>
+              <td>{{ object.description|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">MIME Type</th>
+              <td>{{ object.mime_type|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">File Extension</th>
+              <td>{{ object.file_extension|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">Attachment</th>
+              <td>{% checkmark object.as_attachment %}</td>
+            </tr>
+              <tr>
+                <th scope="row">Data Source</th>
+                <td>
+                  {% if object.data_source %}
+                    <a href="{{ object.data_source.get_absolute_url }}">{{ object.data_source }}</a>
+                  {% else %}
+                    {{ ''|placeholder }}
+                  {% endif %}
+                </td>
+              </tr>
+              <tr>
+                <th scope="row">Data File</th>
+                <td>
+                  {% if object.data_file %}
+                    <a href="{{ object.data_file.get_absolute_url }}">{{ object.data_file }}</a>
+                  {% elif object.data_path %}
+                    <div class="float-end text-warning">
+                      <i class="mdi mdi-alert" title="The data file associated with this object has been deleted."></i>
+                    </div>
+                    {{ object.data_path }}
+                  {% else %}
+                    {{ ''|placeholder }}
+                  {% endif %}
+                </td>
+              </tr>
+              <tr>
+                <th scope="row">Data Synced</th>
+                <td>{{ object.data_synced|placeholder }}</td>
+              </tr>
+          </table>
+        </div>
       </div>
-    </div>
-    {% plugin_left_page object %}
-	</div>
-	<div class="col col-md-7">
-    <div class="card">
-      <h5 class="card-header">
-        Template
-      </h5>
-      <div class="card-body">
-        <pre>{{ object.template_code }}</pre>
+      <div class="card">
+        <h5 class="card-header">Assigned Models</h5>
+        <div class="card-body">
+          <table class="table table-hover attr-table">
+            {% for ct in object.content_types.all %}
+              <tr>
+                <td>{{ ct }}</td>
+              </tr>
+            {% endfor %}
+          </table>
+        </div>
       </div>
+      {% plugin_left_page object %}
+    </div>
+    <div class="col col-md-7">
+      <div class="card">
+        <h5 class="card-header">Template</h5>
+        <div class="card-body">
+          {% include 'inc/sync_warning.html' %}
+          <pre>{{ object.template_code }}</pre>
+        </div>
+      </div>
+      {% plugin_right_page object %}
     </div>
-    {% plugin_right_page object %}
   </div>
-</div>
-<div class="row">
+  <div class="row">
     <div class="col col-md-12">
-        {% plugin_full_width_page object %}
+      {% plugin_full_width_page object %}
     </div>
-</div>
+  </div>
 {% endblock %}

netbox/templates/extras/exporttemplate_list.html

@@ -0,0 +1,10 @@
+{% extends 'generic/object_list.html' %}
+
+{% block bulk_buttons %}
+  {% if perms.extras.sync_configcontext %}
+    <button type="submit" name="_sync" formaction="{% url 'extras:exporttemplate_bulk_sync' %}" class="btn btn-primary btn-sm">
+      <i class="mdi mdi-sync" aria-hidden="true"></i> Sync Data
+    </button>
+  {% endif %}
+  {{ block.super }}
+{% endblock %}

netbox/templates/inc/sync_warning.html

@@ -0,0 +1,13 @@
+{% load buttons %}
+{% load perms %}
+
+{% if object.data_file and object.data_file.last_updated > object.data_synced %}
+  <div class="alert alert-warning" role="alert">
+    <i class="mdi mdi-alert"></i> Data is out of sync with upstream file (<a href="{{ object.data_file.get_absolute_url }}">{{ object.data_file }}</a>).
+    {% if request.user|can_sync:object %}
+      <div class="float-end">
+        {% sync_button object %}
+      </div>
+    {% endif %}
+  </div>
+{% endif %}

netbox/utilities/templates/buttons/sync.html

@@ -0,0 +1,6 @@
+<form action="{{ url }}" method="post">
+  {% csrf_token %}
+  <button type="submit" class="btn btn-primary btn-sm">
+    <i class="mdi mdi-sync" aria-hidden="true"></i> Sync
+  </button>
+</form>

netbox/utilities/templatetags/buttons.py

@@ -46,6 +46,16 @@ def delete_button(instance):
     }
 
 
+@register.inclusion_tag('buttons/sync.html')
+def sync_button(instance):
+    viewname = get_viewname(instance, 'sync')
+    url = reverse(viewname, kwargs={'pk': instance.pk})
+
+    return {
+        'url': url,
+    }
+
+
 #
 # List buttons
 #

netbox/utilities/templatetags/perms.py

@@ -28,3 +28,8 @@ def can_change(user, instance):
 @register.filter()
 def can_delete(user, instance):
     return _check_permission(user, instance, 'delete')
+
+
+@register.filter()
+def can_sync(user, instance):
+    return _check_permission(user, instance, 'sync')