Closes #11559: Implement config template rendering (#11769)

* WIP

* Add config_template field to Device

* Pre-fetch referenced templates

* Correct up_to_date callable

* Add config_template FK to Device

* Update & merge migrations

* Add config_template FK to Platform

* Add tagging support for ConfigTemplate

* Catch exceptions when rendering device templates in UI

* Refactor ConfigTemplate.render()

* Add support for returning plain text content

* Add ConfigTemplate model documentation

* Add feature documentation for config rendering

netbox/extras/api/nested_serializers.py

@@ -7,6 +7,7 @@ from users.api.nested_serializers import NestedUserSerializer
 
 __all__ = [
     'NestedConfigContextSerializer',
+    'NestedConfigTemplateSerializer',
     'NestedCustomFieldSerializer',
     'NestedCustomLinkSerializer',
     'NestedExportTemplateSerializer',
@@ -51,6 +52,14 @@ class NestedConfigContextSerializer(WritableNestedSerializer):
         fields = ['id', 'url', 'display', 'name']
 
 
+class NestedConfigTemplateSerializer(WritableNestedSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='extras-api:configtemplate-detail')
+
+    class Meta:
+        model = models.ConfigTemplate
+        fields = ['id', 'url', 'display', 'name']
+
+
 class NestedExportTemplateSerializer(WritableNestedSerializer):
     url = serializers.HyperlinkedIdentityField(view_name='extras-api:exporttemplate-detail')
 

netbox/extras/api/serializers.py

@@ -16,6 +16,7 @@ from extras.utils import FeatureQuery
 from netbox.api.exceptions import SerializerNotFound
 from netbox.api.fields import ChoiceField, ContentTypeField, SerializedPKRelatedField
 from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer, ValidatedModelSerializer
+from netbox.api.serializers.features import TaggableModelSerializer
 from netbox.constants import NESTED_SERIALIZER_PREFIX
 from tenancy.api.nested_serializers import NestedTenantSerializer, NestedTenantGroupSerializer
 from tenancy.models import Tenant, TenantGroup
@@ -29,6 +30,7 @@ from .nested_serializers import *
 
 __all__ = (
     'ConfigContextSerializer',
+    'ConfigTemplateSerializer',
     'ContentTypeSerializer',
     'CustomFieldSerializer',
     'CustomLinkSerializer',
@@ -383,6 +385,27 @@ class ConfigContextSerializer(ValidatedModelSerializer):
         ]
 
 
+#
+# Config templates
+#
+
+class ConfigTemplateSerializer(TaggableModelSerializer, ValidatedModelSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name='extras-api:configtemplate-detail')
+    data_source = NestedDataSourceSerializer(
+        required=False
+    )
+    data_file = NestedDataFileSerializer(
+        read_only=True
+    )
+
+    class Meta:
+        model = ConfigTemplate
+        fields = [
+            'id', 'url', 'display', 'name', 'description', 'environment_params', 'template_code', 'data_source',
+            'data_path', 'data_file', 'data_synced', 'tags', 'created', 'last_updated',
+        ]
+
+
 #
 # Job Results
 #

netbox/extras/api/urls.py

@@ -14,6 +14,7 @@ router.register('tags', views.TagViewSet)
 router.register('image-attachments', views.ImageAttachmentViewSet)
 router.register('journal-entries', views.JournalEntryViewSet)
 router.register('config-contexts', views.ConfigContextViewSet)
+router.register('config-templates', views.ConfigTemplateViewSet)
 router.register('reports', views.ReportViewSet, basename='report')
 router.register('scripts', views.ScriptViewSet, basename='script')
 router.register('object-changes', views.ObjectChangeViewSet)

netbox/extras/api/views.py

@@ -5,6 +5,7 @@ from rest_framework import status
 from rest_framework.decorators import action
 from rest_framework.exceptions import PermissionDenied
 from rest_framework.permissions import IsAuthenticated
+from rest_framework.renderers import JSONRenderer
 from rest_framework.response import Response
 from rest_framework.routers import APIRootView
 from rest_framework.viewsets import ReadOnlyModelViewSet, ViewSet
@@ -19,10 +20,12 @@ from extras.scripts import get_script, get_scripts, run_script
 from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired
 from netbox.api.features import SyncedDataMixin
 from netbox.api.metadata import ContentTypeMetadata
+from netbox.api.renderers import TextRenderer
 from netbox.api.viewsets import NetBoxModelViewSet
 from utilities.exceptions import RQWorkerNotRunningException
 from utilities.utils import copy_safe_request, count_related
 from . import serializers
+from .nested_serializers import NestedConfigTemplateSerializer
 
 
 class ExtrasRootView(APIRootView):
@@ -157,6 +160,35 @@ class ConfigContextViewSet(SyncedDataMixin, NetBoxModelViewSet):
     filterset_class = filtersets.ConfigContextFilterSet
 
 
+#
+# Config templates
+#
+
+class ConfigTemplateViewSet(SyncedDataMixin, NetBoxModelViewSet):
+    queryset = ConfigTemplate.objects.prefetch_related('data_source', 'data_file')
+    serializer_class = serializers.ConfigTemplateSerializer
+    filterset_class = filtersets.ConfigTemplateFilterSet
+
+    @action(detail=True, methods=['post'], renderer_classes=[JSONRenderer, TextRenderer])
+    def render(self, request, pk):
+        """
+        Render a ConfigTemplate using the context data provided (if any). If the client requests "text/plain" data,
+        return the raw rendered content, rather than serialized JSON.
+        """
+        configtemplate = self.get_object()
+        output = configtemplate.render(context=request.data)
+
+        # If the client has requested "text/plain", return the raw content.
+        if request.accepted_renderer.format == 'txt':
+            return Response(output)
+
+        template_serializer = NestedConfigTemplateSerializer(configtemplate, context={'request': request})
+        return Response({
+            'configtemplate': template_serializer.data,
+            'content': output
+        })
+
+
 #
 # Reports
 #

netbox/extras/filtersets.py

@@ -4,18 +4,19 @@ from django.contrib.contenttypes.models import ContentType
 from django.db.models import Q
 from django.utils.translation import gettext as _
 
-from core.models import DataFile, DataSource
+from core.models import DataSource
 from dcim.models import DeviceRole, DeviceType, Location, Platform, Region, Site, SiteGroup
 from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
 from tenancy.models import Tenant, TenantGroup
 from utilities.filters import ContentTypeFilter, MultiValueCharFilter, MultiValueNumberFilter
 from virtualization.models import Cluster, ClusterGroup, ClusterType
 from .choices import *
+from .filters import TagFilter
 from .models import *
 
-
 __all__ = (
     'ConfigContextFilterSet',
+    'ConfigTemplateFilterSet',
     'ContentTypeFilterSet',
     'CustomFieldFilterSet',
     'CustomLinkFilterSet',
@@ -454,6 +455,34 @@ class ConfigContextFilterSet(ChangeLoggedModelFilterSet):
         )
 
 
+class ConfigTemplateFilterSet(BaseFilterSet):
+    q = django_filters.CharFilter(
+        method='search',
+        label=_('Search'),
+    )
+    data_source_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=DataSource.objects.all(),
+        label=_('Data source (ID)'),
+    )
+    data_file_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=DataSource.objects.all(),
+        label=_('Data file (ID)'),
+    )
+    tag = TagFilter()
+
+    class Meta:
+        model = ConfigTemplate
+        fields = ['id', 'name', 'description', 'data_synced']
+
+    def search(self, queryset, name, value):
+        if not value.strip():
+            return queryset
+        return queryset.filter(
+            Q(name__icontains=value) |
+            Q(description__icontains=value)
+        )
+
+
 #
 # Filter for Local Config Context Data
 #

netbox/extras/forms/bulk_edit.py

@@ -9,6 +9,7 @@ from utilities.forms import (
 
 __all__ = (
     'ConfigContextBulkEditForm',
+    'ConfigTemplateBulkEditForm',
     'CustomFieldBulkEditForm',
     'CustomLinkBulkEditForm',
     'ExportTemplateBulkEditForm',
@@ -201,6 +202,19 @@ class ConfigContextBulkEditForm(BulkEditForm):
     nullable_fields = ('description',)
 
 
+class ConfigTemplateBulkEditForm(BulkEditForm):
+    pk = forms.ModelMultipleChoiceField(
+        queryset=ConfigTemplate.objects.all(),
+        widget=forms.MultipleHiddenInput
+    )
+    description = forms.CharField(
+        max_length=200,
+        required=False
+    )
+
+    nullable_fields = ('description',)
+
+
 class JournalEntryBulkEditForm(BulkEditForm):
     pk = forms.ModelMultipleChoiceField(
         queryset=JournalEntry.objects.all(),

netbox/extras/forms/bulk_import.py

@@ -10,6 +10,7 @@ from extras.utils import FeatureQuery
 from utilities.forms import CSVChoiceField, CSVContentTypeField, CSVModelForm, CSVMultipleContentTypeField, SlugField
 
 __all__ = (
+    'ConfigTemplateImportForm',
     'CustomFieldImportForm',
     'CustomLinkImportForm',
     'ExportTemplateImportForm',
@@ -83,6 +84,15 @@ class ExportTemplateImportForm(CSVModelForm):
         )
 
 
+class ConfigTemplateImportForm(CSVModelForm):
+
+    class Meta:
+        model = ConfigTemplate
+        fields = (
+            'name', 'description', 'environment_params', 'template_code', 'tags',
+        )
+
+
 class SavedFilterImportForm(CSVModelForm):
     content_types = CSVMultipleContentTypeField(
         queryset=ContentType.objects.all(),

netbox/extras/forms/filtersets.py

@@ -20,6 +20,7 @@ from .mixins import SavedFiltersMixin
 
 __all__ = (
     'ConfigContextFilterForm',
+    'ConfigTemplateFilterForm',
     'CustomFieldFilterForm',
     'CustomLinkFilterForm',
     'ExportTemplateFilterForm',
@@ -358,6 +359,27 @@ class ConfigContextFilterForm(SavedFiltersMixin, FilterForm):
     )
 
 
+class ConfigTemplateFilterForm(SavedFiltersMixin, FilterForm):
+    fieldsets = (
+        (None, ('q', 'filter_id', 'tag')),
+        ('Data', ('data_source_id', 'data_file_id')),
+    )
+    data_source_id = DynamicModelMultipleChoiceField(
+        queryset=DataSource.objects.all(),
+        required=False,
+        label=_('Data source')
+    )
+    data_file_id = DynamicModelMultipleChoiceField(
+        queryset=DataFile.objects.all(),
+        required=False,
+        label=_('Data file'),
+        query_params={
+            'source_id': '$data_source_id'
+        }
+    )
+    tag = TagFilterField(ConfigTemplate)
+
+
 class LocalConfigContextFilterForm(forms.Form):
     local_context_data = forms.NullBooleanField(
         required=False,

netbox/extras/forms/model_forms.py

@@ -18,6 +18,7 @@ from virtualization.models import Cluster, ClusterGroup, ClusterType
 
 __all__ = (
     'ConfigContextForm',
+    'ConfigTemplateForm',
     'CustomFieldForm',
     'CustomLinkForm',
     'ExportTemplateForm',
@@ -269,6 +270,34 @@ class ConfigContextForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm):
         return self.cleaned_data
 
 
+class ConfigTemplateForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm):
+    tags = DynamicModelMultipleChoiceField(
+        queryset=Tag.objects.all(),
+        required=False
+    )
+    template_code = forms.CharField(
+        required=False,
+        widget=forms.Textarea(attrs={'class': 'font-monospace'})
+    )
+
+    fieldsets = (
+        ('Config Template', ('name', 'description', 'environment_params', 'tags')),
+        ('Content', ('data_source', 'data_file', 'template_code',)),
+    )
+
+    class Meta:
+        model = ConfigTemplate
+        fields = '__all__'
+
+    def clean(self):
+        super().clean()
+
+        if not self.cleaned_data.get('template_code') and not self.cleaned_data.get('data_file'):
+            raise forms.ValidationError("Must specify either local content or a data file")
+
+        return self.cleaned_data
+
+
 class ImageAttachmentForm(BootstrapMixin, forms.ModelForm):
 
     class Meta:

netbox/extras/graphql/schema.py

@@ -8,6 +8,9 @@ class ExtrasQuery(graphene.ObjectType):
     config_context = ObjectField(ConfigContextType)
     config_context_list = ObjectListField(ConfigContextType)
 
+    config_template = ObjectField(ConfigTemplateType)
+    config_template_list = ObjectListField(ConfigTemplateType)
+
     custom_field = ObjectField(CustomFieldType)
     custom_field_list = ObjectListField(CustomFieldType)
 

netbox/extras/graphql/types.py

@@ -4,6 +4,7 @@ from netbox.graphql.types import BaseObjectType, ObjectType
 
 __all__ = (
     'ConfigContextType',
+    'ConfigTemplateType',
     'CustomFieldType',
     'CustomLinkType',
     'ExportTemplateType',
@@ -24,6 +25,14 @@ class ConfigContextType(ObjectType):
         filterset_class = filtersets.ConfigContextFilterSet
 
 
+class ConfigTemplateType(ObjectType):
+
+    class Meta:
+        model = models.ConfigTemplate
+        fields = '__all__'
+        filterset_class = filtersets.ConfigTemplateFilterSet
+
+
 class CustomFieldType(ObjectType):
 
     class Meta:

netbox/extras/migrations/0086_configtemplate.py

@@ -0,0 +1,34 @@
+from django.db import migrations, models
+import django.db.models.deletion
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+        ('extras', '0085_synced_data'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ConfigTemplate',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)),
+                ('created', models.DateTimeField(auto_now_add=True, null=True)),
+                ('last_updated', models.DateTimeField(auto_now=True, null=True)),
+                ('data_path', models.CharField(blank=True, editable=False, max_length=1000)),
+                ('data_synced', models.DateTimeField(blank=True, editable=False, null=True)),
+                ('name', models.CharField(max_length=100)),
+                ('description', models.CharField(blank=True, max_length=200)),
+                ('template_code', models.TextField()),
+                ('environment_params', models.JSONField(blank=True, null=True)),
+                ('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')),
+                ('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')),
+                ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')),
+            ],
+            options={
+                'ordering': ('name',),
+            },
+        ),
+    ]

netbox/extras/models/__init__.py

@@ -1,5 +1,5 @@
 from .change_logging import ObjectChange
-from .configcontexts import ConfigContext, ConfigContextModel
+from .configs import *
 from .customfields import CustomField
 from .models import *
 from .search import *
@@ -12,6 +12,7 @@ __all__ = (
     'ConfigContext',
     'ConfigContextModel',
     'ConfigRevision',
+    'ConfigTemplate',
     'CustomField',
     'CustomLink',
     'ExportTemplate',

netbox/extras/models/configs.py

@@ -3,15 +3,21 @@ from django.core.validators import ValidationError
 from django.db import models
 from django.urls import reverse
 from django.utils import timezone
+from django.utils.translation import gettext as _
+from jinja2.loaders import BaseLoader
+from jinja2.sandbox import SandboxedEnvironment
 
 from extras.querysets import ConfigContextQuerySet
+from netbox.config import get_config
 from netbox.models import ChangeLoggedModel
-from netbox.models.features import SyncedDataMixin
+from netbox.models.features import ExportTemplatesMixin, SyncedDataMixin, TagsMixin
+from utilities.jinja2 import ConfigTemplateLoader
 from utilities.utils import deepmerge
 
 __all__ = (
     'ConfigContext',
     'ConfigContextModel',
+    'ConfigTemplate',
 )
 
 
@@ -182,3 +188,77 @@ class ConfigContextModel(models.Model):
             raise ValidationError(
                 {'local_context_data': 'JSON data must be in object form. Example: {"foo": 123}'}
             )
+
+
+#
+# Config templates
+#
+
+class ConfigTemplate(SyncedDataMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel):
+    name = models.CharField(
+        max_length=100
+    )
+    description = models.CharField(
+        max_length=200,
+        blank=True
+    )
+    template_code = models.TextField(
+        help_text=_('Jinja2 template code.')
+    )
+    environment_params = models.JSONField(
+        blank=True,
+        null=True
+    )
+
+    class Meta:
+        ordering = ('name',)
+
+    def __str__(self):
+        return self.name
+
+    def get_absolute_url(self):
+        return reverse('extras:configtemplate', args=[self.pk])
+
+    def sync_data(self):
+        """
+        Synchronize template content from the designated DataFile (if any).
+        """
+        self.template_code = self.data_file.data_as_string
+        self.data_synced = timezone.now()
+
+    def render(self, context=None):
+        """
+        Render the contents of the template.
+        """
+        context = context or {}
+
+        # Initialize the Jinja2 environment and instantiate the Template
+        environment = self._get_environment()
+        if self.data_file:
+            template = environment.get_template(self.data_file.path)
+        else:
+            template = environment.from_string(self.template_code)
+
+        output = template.render(**context)
+
+        # Replace CRLF-style line terminators
+        return output.replace('\r\n', '\n')
+
+    def _get_environment(self):
+        """
+        Instantiate and return a Jinja2 environment suitable for rendering the ConfigTemplate.
+        """
+        # Initialize the template loader & cache the base template code (if applicable)
+        if self.data_file:
+            loader = ConfigTemplateLoader(data_source=self.data_source)
+            loader.cache_templates({
+                self.data_file.path: self.template_code
+            })
+        else:
+            loader = BaseLoader()
+
+        # Initialize the environment
+        environment = SandboxedEnvironment(loader=loader)
+        environment.filters.update(get_config().JINJA2_FILTERS)
+
+        return environment

netbox/extras/tables/tables.py

@@ -8,6 +8,7 @@ from .template_code import *
 
 __all__ = (
     'ConfigContextTable',
+    'ConfigTemplateTable',
     'CustomFieldTable',
     'CustomLinkTable',
     'ExportTemplateTable',
@@ -223,6 +224,34 @@ class ConfigContextTable(NetBoxTable):
         default_columns = ('pk', 'name', 'weight', 'is_active', 'is_synced', 'description')
 
 
+class ConfigTemplateTable(NetBoxTable):
+    name = tables.Column(
+        linkify=True
+    )
+    data_source = tables.Column(
+        linkify=True
+    )
+    data_file = tables.Column(
+        linkify=True
+    )
+    is_synced = columns.BooleanColumn(
+        verbose_name='Synced'
+    )
+    tags = columns.TagColumn(
+        url_name='extras:configtemplate_list'
+    )
+
+    class Meta(NetBoxTable.Meta):
+        model = ConfigTemplate
+        fields = (
+            'pk', 'id', 'name', 'description', 'data_source', 'data_file', 'data_synced', 'created', 'last_updated',
+            'tags',
+        )
+        default_columns = (
+            'pk', 'name', 'description', 'is_synced',
+        )
+
+
 class ObjectChangeTable(NetBoxTable):
     time = tables.DateTimeColumn(
         linkify=True,

netbox/extras/urls.py

@@ -64,6 +64,14 @@ urlpatterns = [
     path('config-contexts/sync/', views.ConfigContextBulkSyncDataView.as_view(), name='configcontext_bulk_sync'),
     path('config-contexts/<int:pk>/', include(get_model_urls('extras', 'configcontext'))),
 
+    # Config templates
+    path('config-templates/', views.ConfigTemplateListView.as_view(), name='configtemplate_list'),
+    path('config-templates/add/', views.ConfigTemplateEditView.as_view(), name='configtemplate_add'),
+    path('config-templates/edit/', views.ConfigTemplateBulkEditView.as_view(), name='configtemplate_bulk_edit'),
+    path('config-templates/delete/', views.ConfigTemplateBulkDeleteView.as_view(), name='configtemplate_bulk_delete'),
+    path('config-templates/sync/', views.ConfigTemplateBulkSyncDataView.as_view(), name='configtemplate_bulk_sync'),
+    path('config-templates/<int:pk>/', include(get_model_urls('extras', 'configtemplate'))),
+
     # Image attachments
     path('image-attachments/add/', views.ImageAttachmentEditView.as_view(), name='imageattachment_add'),
     path('image-attachments/<int:pk>/', include(get_model_urls('extras', 'imageattachment'))),

netbox/extras/views.py

@@ -452,6 +452,58 @@ class ObjectConfigContextView(generic.ObjectView):
         }
 
 
+#
+# Config templates
+#
+
+class ConfigTemplateListView(generic.ObjectListView):
+    queryset = ConfigTemplate.objects.all()
+    filterset = filtersets.ConfigTemplateFilterSet
+    filterset_form = forms.ConfigTemplateFilterForm
+    table = tables.ConfigTemplateTable
+    template_name = 'extras/configtemplate_list.html'
+    actions = ('add', 'import', 'export', 'bulk_edit', 'bulk_delete', 'bulk_sync')
+
+
+@register_model_view(ConfigTemplate)
+class ConfigTemplateView(generic.ObjectView):
+    queryset = ConfigTemplate.objects.all()
+
+
+@register_model_view(ConfigTemplate, 'edit')
+class ConfigTemplateEditView(generic.ObjectEditView):
+    queryset = ConfigTemplate.objects.all()
+    form = forms.ConfigTemplateForm
+
+
+@register_model_view(ConfigTemplate, 'delete')
+class ConfigTemplateDeleteView(generic.ObjectDeleteView):
+    queryset = ConfigTemplate.objects.all()
+
+
+class ConfigTemplateBulkImportView(generic.BulkImportView):
+    queryset = ConfigTemplate.objects.all()
+    model_form = forms.ConfigTemplateImportForm
+    table = tables.ConfigTemplateTable
+
+
+class ConfigTemplateBulkEditView(generic.BulkEditView):
+    queryset = ConfigTemplate.objects.all()
+    filterset = filtersets.ConfigTemplateFilterSet
+    table = tables.ConfigTemplateTable
+    form = forms.ConfigTemplateBulkEditForm
+
+
+class ConfigTemplateBulkDeleteView(generic.BulkDeleteView):
+    queryset = ConfigTemplate.objects.all()
+    filterset = filtersets.ConfigTemplateFilterSet
+    table = tables.ConfigTemplateTable
+
+
+class ConfigTemplateBulkSyncDataView(generic.BulkSyncDataView):
+    queryset = ConfigTemplate.objects.all()
+
+
 #
 # Change logging
 #