Closes #12906: Make boto3 & dulwich libraries optional (#13324)

* Initial work on #12906

* Catch import errors during backend init

* Tweak error message

* Update requirements & add note to docs

netbox/core/data_backends.py

@@ -6,13 +6,9 @@ from contextlib import contextmanager
 from pathlib import Path
 from urllib.parse import urlparse
 
-import boto3
-from botocore.config import Config as Boto3Config
 from django import forms
 from django.conf import settings
 from django.utils.translation import gettext as _
-from dulwich import porcelain
-from dulwich.config import ConfigDict
 
 from netbox.registry import registry
 from .choices import DataSourceTypeChoices
@@ -43,9 +39,20 @@ class DataBackend:
     parameters = {}
     sensitive_parameters = []
 
+    # Prevent Django's template engine from calling the backend
+    # class when referenced via DataSource.backend_class
+    do_not_call_in_templates = True
+
     def __init__(self, url, **kwargs):
         self.url = url
         self.params = kwargs
+        self.config = self.init_config()
+
+    def init_config(self):
+        """
+        Hook to initialize the instance's configuration.
+        """
+        return
 
     @property
     def url_scheme(self):
@@ -58,6 +65,7 @@ class DataBackend:
 
 @register_backend(DataSourceTypeChoices.LOCAL)
 class LocalBackend(DataBackend):
+
     @contextmanager
     def fetch(self):
         logger.debug(f"Data source type is local; skipping fetch")
@@ -89,14 +97,28 @@ class GitBackend(DataBackend):
     }
     sensitive_parameters = ['password']
 
+    def init_config(self):
+        from dulwich.config import ConfigDict
+
+        # Initialize backend config
+        config = ConfigDict()
+
+        # Apply HTTP proxy (if configured)
+        if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'):
+            if proxy := settings.HTTP_PROXIES.get(self.url_scheme):
+                config.set("http", "proxy", proxy)
+
+        return config
+
     @contextmanager
     def fetch(self):
+        from dulwich import porcelain
+
         local_path = tempfile.TemporaryDirectory()
 
-        config = ConfigDict()
         clone_args = {
             "branch": self.params.get('branch'),
-            "config": config,
+            "config": self.config,
             "depth": 1,
             "errstream": porcelain.NoneStream(),
             "quiet": True,
@@ -110,10 +132,6 @@ class GitBackend(DataBackend):
                 }
             )
 
-        if settings.HTTP_PROXIES and self.url_scheme in ('http', 'https'):
-            if proxy := settings.HTTP_PROXIES.get(self.url_scheme):
-                config.set("http", "proxy", proxy)
-
         logger.debug(f"Cloning git repo: {self.url}")
         try:
             porcelain.clone(self.url, local_path.name, **clone_args)
@@ -141,15 +159,20 @@ class S3Backend(DataBackend):
 
     REGION_REGEX = r's3\.([a-z0-9-]+)\.amazonaws\.com'
 
-    @contextmanager
-    def fetch(self):
-        local_path = tempfile.TemporaryDirectory()
+    def init_config(self):
+        from botocore.config import Config as Boto3Config
 
-        # Build the S3 configuration
-        s3_config = Boto3Config(
+        # Initialize backend config
+        return Boto3Config(
             proxies=settings.HTTP_PROXIES,
         )
 
+    @contextmanager
+    def fetch(self):
+        import boto3
+
+        local_path = tempfile.TemporaryDirectory()
+
         # Initialize the S3 resource and bucket
         aws_access_key_id = self.params.get('aws_access_key_id')
         aws_secret_access_key = self.params.get('aws_secret_access_key')
@@ -158,7 +181,7 @@ class S3Backend(DataBackend):
             region_name=self._region_name,
             aws_access_key_id=aws_access_key_id,
             aws_secret_access_key=aws_secret_access_key,
-            config=s3_config
+            config=self.config
         )
         bucket = s3.Bucket(self._bucket_name)
 

netbox/core/models/data.py

@@ -104,6 +104,10 @@ class DataSource(JobsMixin, PrimaryModel):
     def url_scheme(self):
         return urlparse(self.source_url).scheme.lower()
 
+    @property
+    def backend_class(self):
+        return registry['data_backends'].get(self.type)
+
     @property
     def is_local(self):
         return self.type == DataSourceTypeChoices.LOCAL
@@ -139,17 +143,15 @@ class DataSource(JobsMixin, PrimaryModel):
         )
 
     def get_backend(self):
-        backend_cls = registry['data_backends'].get(self.type)
         backend_params = self.parameters or {}
-
-        return backend_cls(self.source_url, **backend_params)
+        return self.backend_class(self.source_url, **backend_params)
 
     def sync(self):
         """
         Create/update/delete child DataFiles as necessary to synchronize with the remote source.
         """
         if self.status == DataSourceStatusChoices.SYNCING:
-            raise SyncError(f"Cannot initiate sync; syncing already in progress.")
+            raise SyncError("Cannot initiate sync; syncing already in progress.")
 
         # Emit the pre_sync signal
         pre_sync.send(sender=self.__class__, instance=self)
@@ -158,7 +160,12 @@ class DataSource(JobsMixin, PrimaryModel):
         DataSource.objects.filter(pk=self.pk).update(status=self.status)
 
         # Replicate source data locally
-        backend = self.get_backend()
+        try:
+            backend = self.get_backend()
+        except ModuleNotFoundError as e:
+            raise SyncError(
+                f"There was an error initializing the backend. A dependency needs to be installed: {e}"
+            )
         with backend.fetch() as local_path:
 
             logger.debug(f'Syncing files from source root {local_path}')

netbox/templates/core/datasource.html

@@ -85,24 +85,26 @@
       <div class="card">
         <h5 class="card-header">{% trans "Backend" %}</h5>
         <div class="card-body">
-          <table class="table table-hover attr-table">
-            {% for name, field in object.get_backend.parameters.items %}
-              <tr>
-                <th scope="row">{{ field.label }}</th>
-                {% if name in object.get_backend.sensitive_parameters and not perms.core.change_datasource %}
-                  <td>********</td>
-                {% else %}
-                  <td>{{ object.parameters|get_key:name|placeholder }}</td>
-                {% endif %}
-              </tr>
-            {% empty %}
-              <tr>
-                <td colspan="2" class="text-muted">
-                  {% trans "No parameters defined" %}
-                </td>
-              </tr>
-            {% endfor %}
-          </table>
+          {% with backend=object.backend_class %}
+            <table class="table table-hover attr-table">
+              {% for name, field in backend.parameters.items %}
+                <tr>
+                  <th scope="row">{{ field.label }}</th>
+                  {% if name in backend.sensitive_parameters and not perms.core.change_datasource %}
+                    <td>********</td>
+                  {% else %}
+                    <td>{{ object.parameters|get_key:name|placeholder }}</td>
+                  {% endif %}
+                </tr>
+              {% empty %}
+                <tr>
+                  <td colspan="2" class="text-muted">
+                    {% trans "No parameters defined" %}
+                  </td>
+                </tr>
+              {% endfor %}
+            </table>
+            {% endwith %}
         </div>
       </div>
       {% include 'inc/panels/related_objects.html' %}